From 8b8cb3fadbcd1eb631fd6c9499f34db600e77d7a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:35:38 +0000 Subject: [PATCH 01/23] =?UTF-8?q?refactor(stage3):=20select=20over=20execu?= =?UTF-8?q?tion=20contexts:=20x=5F{=CE=BA,v},=20y=5F{=CE=BA,u,c,i},=20h=5F?= =?UTF-8?q?v=20and=20constraint=20(R);=20asap-cost=20computes=20once=20per?= =?UTF-8?q?=20context=20(#1207=20PR=207)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/cost/src/node/usage/mod.rs | 154 ++-- crates/cost/src/node/usage/tests.rs | 108 ++- .../mod.rs} | 329 ++++----- .../cost/src/plan/execution_demand/tests.rs | 317 +++++++++ .../tests/stage3_objective.rs | 87 +++ crates/planner/tests/runtime_install.rs | 60 +- crates/stage3-plan-selection/src/cost.rs | 669 ++++++++++++------ .../stage3-plan-selection/src/cost/tests.rs | 224 ++++++ crates/stage3-plan-selection/src/lib.rs | 68 +- .../src/milp/execution_contexts.rs | 136 +++- .../stage3-plan-selection/src/milp/graph.rs | 28 +- crates/stage3-plan-selection/src/milp/mod.rs | 9 +- .../src/milp/model/mod.rs | 550 +++++++------- .../src/milp/model/tests.rs | 328 ++++++--- .../src/milp/upper_bound.rs | 172 +++-- crates/stage3-plan-selection/src/plan.rs | 108 ++- crates/stage3-plan-selection/src/report.rs | 46 +- .../stage3-plan-selection/src/report/tests.rs | 30 + crates/stage3-plan-selection/src/retention.rs | 67 +- .../stage3-plan-selection/src/selectable.rs | 372 +++++++--- .../src/ir/operator/tumbling_window_chain.rs | 4 +- crates/types/src/stages/physical/candidate.rs | 59 +- .../stages/physical/memo_candidates/tests.rs | 64 ++ .../design_docs/proposals/planner-layering.md | 27 +- .../asap-aware-mapping-architecture.md | 2 +- 25 files changed, 2851 insertions(+), 1167 deletions(-) rename crates/cost/src/plan/{execution_demand.rs => execution_demand/mod.rs} (69%) create mode 100644 crates/cost/src/plan/execution_demand/tests.rs create mode 100644 crates/stage3-plan-selection/src/cost/tests.rs create mode 100644 crates/stage3-plan-selection/src/report/tests.rs diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 668681e7c..af859b6c5 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -123,8 +123,13 @@ pub struct ProducerOutput<'a> { /// column) values unchanged ([`unchanged_source_columns`]). pub unchanged_columns: BTreeSet<&'a str>, /// The tumbling windows its rows span: those a `TimeRange` selects - /// from a chain, 1 for any other node. + /// from a chain (through `TimeShift`s), 1 for any other node. pub tumbling_windows: u64, + /// The tumbling-window width of the chain whose states its rows are + /// read from: a chain's own, kept through the `TimeShift`s over it, + /// so a `TimeRange` over them selects windows; `None` for any other + /// node. + pub chain_tumbling_window_width_ms: Option, } /// What one node uses and outputs ([`node_resource_usage_and_cost`]): its @@ -146,8 +151,13 @@ pub struct NodeResourceUsageAndCost<'a> { /// column) values unchanged ([`unchanged_source_columns`]). pub unchanged_columns: BTreeSet<&'a str>, /// The tumbling windows its rows span: those a `TimeRange` selects - /// from a chain, 1 for any other node. + /// from a chain (through `TimeShift`s), 1 for any other node. pub tumbling_windows: u64, + /// The tumbling-window width of the chain whose states its rows are + /// read from: a chain's own, kept through the `TimeShift`s over it, + /// so a `TimeRange` over them selects windows; `None` for any other + /// node. + pub chain_tumbling_window_width_ms: Option, /// Work of one execution, in ms on one core. pub work_ms_per_execution: f64, /// CPU operations per second. @@ -187,6 +197,7 @@ impl<'a> NodeResourceUsageAndCost<'a> { family: self.family, unchanged_columns: self.unchanged_columns.clone(), tumbling_windows: self.tumbling_windows, + chain_tumbling_window_width_ms: self.chain_tumbling_window_width_ms, } } } @@ -249,9 +260,10 @@ pub fn maintained_window_ms<'o, N: Copy + Ord, C: 'o>( /// most offset + range its consumers read ([`read_back_ms`] without a span; /// `None`: it reads what it retains). A materialized node keeps what its /// `retention_ms` ($h_v$) covers: a tumbling-window chain its windows in -/// it, a query-time `Scan` its raw rows in it, any other view materialized -/// at ingestion time one output per refresh in it (one second of rows -/// when it maintains no window). The formulas' CPU is scaled by +/// it, a query-time `Scan` and raw rows materialized at ingestion time (a +/// `Scan`, or a `TimeShift` by an offset over one) their raw rows in it, +/// any other view materialized at ingestion time one output per refresh in +/// it (one second of rows when it maintains no window). The formulas' CPU is scaled by /// `settings.frozen_calibration_factor` (1 when unset). /// /// [`executions_per_second`]: crate::plan::execution_demand::executions_per_second @@ -307,13 +319,16 @@ pub fn node_resource_usage_and_cost<'a>( .max(1); let inputs: Vec = producers.iter().map(|p| p.output).collect(); // The tumbling windows its rows span: those a `TimeRange` selects from - // a chain, 1 otherwise. + // a chain (through `TimeShift`s, which pass them on), 1 otherwise. let tumbling_windows = match &node.payload { Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) - if chain_width_ms(producers).is_some() => + if chain_tumbling_window_width_ms(producers).is_some() => { selected_tumbling_windows(range, producers)? } + Payload::NonASAP(NonASAPOp::TimeShift { .. }) => producers + .first() + .map_or(1, |producer| producer.tumbling_windows), _ => 1, }; let sources: BTreeSet<&str> = match &node.payload { @@ -328,7 +343,43 @@ pub fn node_resource_usage_and_cost<'a>( // The rows per ms its sources receive, which its scans and time // ranges read, when the data declares an arrival rate. let rows_per_ms = source_rows_per_ms(&sources, data); + // Raw rows materialized at ingestion time: a `Scan`, or a `TimeShift` + // by an offset over one, across evaluations. It is a time-indexed store + // its consumers read back in, retaining `retention_ms` ($h_v$) of rows, + // which behind an offset includes the offset. + let raw_rows_materialized_at_ingestion = ingestion + && matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) + && match &node.payload { + Payload::NonASAP(NonASAPOp::Scan { .. }) => true, + Payload::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + shift.at.is_none() + && producers.first().is_some_and(|producer| { + matches!(producer.payload, Payload::NonASAP(NonASAPOp::Scan { .. })) + }) + } + _ => false, + }; + // The raw rows of its sources back `lookback_ms` (every row held + // without one): by arrival rate, at most the rows they hold. + let retained_raw_rows = |lookback_ms: Option| -> Option { + let by_rate = rows_per_ms + .zip(lookback_ms) + .map(|(rows_per_ms, lookback_ms)| (rows_per_ms * lookback_ms as f64).round() as u64); + let rows_held: Option = sources + .iter() + .map(|source| data.sources.get(*source).and_then(|facts| facts.rows_held)) + .sum(); + match (by_rate, rows_held) { + (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), + (by_rate, rows_held) => by_rate.or(rows_held), + } + .map(|rows| rows.max(1)) + }; // The fact a rate needs, named when it is missing. + let retained_without_a_bound = |source: &str| AnalyticalCostError::MissingSourceFact { + source_name: source.to_string(), + missing: "its rows_held (raw data retained without a bound)", + }; let missing_rate = |source: &str| AnalyticalCostError::MissingSourceFact { source_name: source.to_string(), missing: "an arrival rate (the arrival's rows_per_second, or the \ @@ -463,12 +514,8 @@ pub fn node_resource_usage_and_cost<'a>( // materialized across evaluations, keeps: back to the // oldest row read, its `retention_ms` ($h_v$; an offset // window keeps more than it reads), at most those held. - if !ingestion || matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) { - let retained_rows = match (by_rate(node.retention_ms), rows_held) { - (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), - (by_rate, rows_held) => by_rate.or(rows_held), - }; - raw_retained_rows = retained_rows.map(|rows| rows.max(1)); + if !ingestion || raw_rows_materialized_at_ingestion { + raw_retained_rows = retained_raw_rows(node.retention_ms); } let by_rate = by_rate(lookback); let rows = match (ingestion, by_rate, rows_held) { @@ -502,14 +549,9 @@ pub fn node_resource_usage_and_cost<'a>( ); // Raw data materialized at ingestion time: its consumers // read the rows it retains, not one second of them. - let out = match (ingestion, node.lifetime) { - (true, Lifetime::AcrossEvaluations(_)) => edge(raw_retained_rows.ok_or( - AnalyticalCostError::MissingSourceFact { - source_name: source.to_string(), - missing: "its rows_held (raw data retained without a bound)", - }, - )?)?, - _ => out, + let out = match raw_rows_materialized_at_ingestion { + true => edge(raw_retained_rows.ok_or(retained_without_a_bound(source))?)?, + false => out, }; (out, estimate, format!("scan {} samples", out.rows)) } @@ -602,17 +644,33 @@ pub fn node_resource_usage_and_cost<'a>( ); (out, estimate, format!("limit to {} rows", out.rows)) } - NonASAPOp::TimeShift { .. } => ( - input, - estimate_operator( - PhysicalOperator::TimeShift, - OperatorStatistics::PassThrough { - edges: unary(input), - }, - ), - format!("time shift: pass {} rows, free", input.rows), - ), - NonASAPOp::TimeRange { range, .. } if chain_width_ms(producers).is_some() => { + NonASAPOp::TimeShift { .. } => { + // Raw rows materialized at ingestion time: like a + // materialized `Scan`'s, its consumers read the rows it + // retains (its retention, its offset included), not one + // second of them. + let out = match raw_rows_materialized_at_ingestion { + true => { + raw_retained_rows = retained_raw_rows(node.retention_ms); + let source = sources.first().copied().unwrap_or("(none)"); + edge(raw_retained_rows.ok_or(retained_without_a_bound(source))?)? + } + false => input, + }; + ( + out, + estimate_operator( + PhysicalOperator::TimeShift, + OperatorStatistics::PassThrough { + edges: unary(input), + }, + ), + format!("time shift: pass {} rows, free", input.rows), + ) + } + NonASAPOp::TimeRange { range, .. } + if chain_tumbling_window_width_ms(producers).is_some() => + { // Over a chain, the windows inside the range: one // window's states per window. let windows = tumbling_windows; @@ -929,8 +987,8 @@ pub fn node_resource_usage_and_cost<'a>( } (0, medium) } - // Raw data materialized at ingestion time: the rows it retains. - (true, Lifetime::AcrossEvaluations(medium), Payload::NonASAP(NonASAPOp::Scan { .. })) => { + // Raw rows materialized at ingestion time: the rows it retains. + (_, Lifetime::AcrossEvaluations(medium), _) if raw_rows_materialized_at_ingestion => { (edge(raw_retained_rows.unwrap_or(out.rows))?.bytes, medium) } // Any other view materialized at ingestion time, a chain included: @@ -997,12 +1055,21 @@ pub fn node_resource_usage_and_cost<'a>( if cost.is_some_and(|cost| !cost.is_finite()) { return Err(AnalyticalCostError::Overflow); } + let chain_tumbling_window_width_ms = match &node.payload { + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(*tumbling_window_width_ms), + Payload::NonASAP(NonASAPOp::TimeShift { .. }) => chain_tumbling_window_width_ms(producers), + _ => None, + }; Ok(NodeResourceUsageAndCost { output: out, sources, family, unchanged_columns, tumbling_windows, + chain_tumbling_window_width_ms, work_ms_per_execution: node_ms, cpu_ops_per_second: estimate.cpu_ops() * per_second, scanned_bytes_per_second: estimate.scan_bytes() as f64 * per_second, @@ -1036,16 +1103,11 @@ pub(crate) fn cost_source(per_node: &BTreeMap) -> } } -/// The tumbling-window width of the chain `producers` starts with, if it -/// is one: a `TimeRange` over it selects windows. -fn chain_width_ms(producers: &[ProducerOutput<'_>]) -> Option { - match producers.first()?.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(*tumbling_window_width_ms), - _ => None, - } +/// The tumbling-window width of the chain the first of `producers` reads +/// from, if any ([`ProducerOutput::chain_tumbling_window_width_ms`]): a +/// `TimeRange` over it selects windows. +fn chain_tumbling_window_width_ms(producers: &[ProducerOutput<'_>]) -> Option { + producers.first()?.chain_tumbling_window_width_ms } /// The windows a `TimeRange` of `range` selects from the chain @@ -1054,7 +1116,9 @@ fn selected_tumbling_windows( range: &std::time::Duration, producers: &[ProducerOutput<'_>], ) -> Result { - let width_ms = chain_width_ms(producers).unwrap_or(1).max(1); + let width_ms = chain_tumbling_window_width_ms(producers) + .unwrap_or(1) + .max(1); Ok(u64::try_from(range.as_millis() / u128::from(width_ms)) .map_err(|_| AnalyticalCostError::Overflow)? .max(1)) diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 1955c1e6b..a13acb752 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -58,6 +58,7 @@ fn a_merge_over_a_chain_s_windows_outputs_one_state_per_group() { family: StateFamily::Exact(&ExactKind::Sum), unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: Some(60_000), }; let windows = node(Payload::NonASAP(NonASAPOp::TimeRange { range: Duration::from_secs(300), @@ -94,6 +95,78 @@ fn a_merge_over_a_chain_s_windows_outputs_one_state_per_group() { ); } +/// o11y q21's shape, `sum(rate(m[5m] offset 1h))` over a chain of 60 s +/// windows: a `TimeShift` by 1 h between the chain and the 5m +/// `TimeRange`, or above the `TimeRange`, changes nothing. The range +/// selects five windows of 1000 states, and the merge over them reads +/// 5000 states into 1000, as with no shift (it read 1000, one window). +#[test] +fn a_time_shift_over_a_chain_or_its_windows_selects_what_the_range_does() { + use asap_types::ir::operator::TimeShift; + let settings = CostInputs::default(); + let data = DataWorkload::default(); + let chain = node(Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + })); + let shift = node(Payload::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at: None, + }, + child: 0, + })); + let range = node(Payload::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: 0, + })); + let merge = node(Payload::ASAP(ASAPOp::SummaryMerge { + children: vec![0], + group_by: None, + })); + // The merge's cost detail and output over `path`, costed bottom-up + // from the chain's one window of 1000 states. + let merge_over = |path: &[&PhysicalASAPDAGNode]| -> (String, EdgeStatistics) { + let mut below = ProducerOutput { + payload: &chain.payload, + output_schema: &chain.output_schema, + output: EdgeStatistics { + rows: 1_000, + bytes: 16_000, + }, + sources: BTreeSet::new(), + family: StateFamily::Exact(&ExactKind::Sum), + unchanged_columns: BTreeSet::new(), + tumbling_windows: 1, + chain_tumbling_window_width_ms: Some(60_000), + }; + for &step in path { + let usage = + node_resource_usage_and_cost(step, &[below], (1, 1.0), None, &data, &settings) + .unwrap(); + below = produced(step, &usage); + } + let usage = + node_resource_usage_and_cost(&merge, &[below], (1, 1.0), None, &data, &settings) + .unwrap(); + (usage.cost.detail, usage.output) + }; + let direct = merge_over(&[&range]); + assert!( + direct.0.starts_with("merge 5000 states into 1000;"), + "{}", + direct.0 + ); + for path in [[&shift, &range], [&range, &shift]] { + assert_eq!(merge_over(&path), direct); + } +} + /// A query-time node holds its state only /// while it computes: bytes × its work per execution × its executions /// per second, with no query latency. A KLL state of 2 MB whose own @@ -120,6 +193,7 @@ fn query_time_memory_is_held_only_while_the_node_computes() { family: StateFamily::Exact(&ExactKind::Sum), unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: None, }; let merge = PhysicalASAPDAGNode { lifetime: Lifetime::OneEvaluation(StorageMedium::Memory), @@ -218,17 +292,21 @@ fn a_query_time_scan_reads_what_its_consumers_read_and_keeps_its_retention() { assert_eq!(kept_bytes, usage.output.bytes * 3_601); } -/// An ingestion-time view of raw rows with no window below it (a -/// `TimeShift` by 90 s over the arriving rows) keeps what its +/// Raw rows materialized at ingestion time behind an offset (a +/// `TimeShift` by 90 s over the arriving rows) keep what their /// `retention_ms` covers, its offset included (a 5-minute read is retained -/// for 390 s, [`read_back_ms`]): one second of rows per second, and the -/// second being built. Over 1,000 rows per second of 64 bytes, a 390 s -/// retention keeps 64,000 × 391 bytes, not one second's 64,000 × 2; a -/// 690 s one keeps 64,000 × 691. +/// for 390 s, [`read_back_ms`]), at the source's arrival rate: 1,000 rows +/// per second retained 390 s keep 390,000 rows, not one second's 1,000; a +/// 690 s retention keeps 690,000. #[test] fn an_ingestion_time_view_of_raw_rows_keeps_the_rows_its_retention_covers() { + use asap_types::ir::schema::{DataType, Field}; + use asap_types::workload::DurationMs; let settings = CostInputs::default(); - let data = DataWorkload::default(); + let mut data = DataWorkload::sampled_every(["m".to_string()], DurationMs(1_000)); + for facts in data.sources.values_mut() { + facts.series_count = Some(1_000); + } let scan = node(Payload::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { metric: "m".into() }, predicates: vec![], @@ -241,16 +319,22 @@ fn an_ingestion_time_view_of_raw_rows_keeps_the_rows_its_retention_covers() { rows: 1_000, bytes: 64_000, }, - sources: BTreeSet::new(), + sources: ["m"].into(), family: StateFamily::None, unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: None, }; + let row_schema = Schema::new(vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]); let kept_bytes = |retention_ms: u64| { let shift = PhysicalASAPDAGNode { output_state: ExecutionDataState::INGESTION_ROWS, lifetime: Lifetime::AcrossEvaluations(StorageMedium::Memory), retention_ms: Some(retention_ms), + output_schema: row_schema.clone(), ..node(Payload::NonASAP(NonASAPOp::TimeShift { shift: asap_types::ir::operator::arguments::TimeShift { offset_ms: 90_000, @@ -270,8 +354,10 @@ fn an_ingestion_time_view_of_raw_rows_keeps_the_rows_its_retention_covers() { .unwrap() .peak_memory_bytes }; - assert_eq!(kept_bytes(390_000), 64_000 * 391); - assert_eq!(kept_bytes(690_000), 64_000 * 691); + let row_width_bytes = row_bytes(&row_schema); + assert!(row_width_bytes > 0); + assert_eq!(kept_bytes(390_000), 390_000 * row_width_bytes); + assert_eq!(kept_bytes(690_000), 690_000 * row_width_bytes); } /// A view materialized across evaluations without a retention is read @@ -299,6 +385,7 @@ fn a_view_without_retention_is_not_costed() { family: StateFamily::None, unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: None, }; let shift = PhysicalASAPDAGNode { output_state: ExecutionDataState::INGESTION_ROWS, @@ -374,6 +461,7 @@ fn a_view_over_an_ingestion_time_chain_keeps_one_output_per_chain_window() { family: StateFamily::Exact(&ExactKind::Sum), unchanged_columns: BTreeSet::new(), tumbling_windows: 1, + chain_tumbling_window_width_ms: Some(60_000), }; let usage = node_resource_usage_and_cost( &shift, diff --git a/crates/cost/src/plan/execution_demand.rs b/crates/cost/src/plan/execution_demand/mod.rs similarity index 69% rename from crates/cost/src/plan/execution_demand.rs rename to crates/cost/src/plan/execution_demand/mod.rs index b939d8fa6..dc022f87f 100644 --- a/crates/cost/src/plan/execution_demand.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -8,10 +8,14 @@ //! to cost once per request, as `computations` does. use crate::node::analytical::AnalyticalCostError; +use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; +use asap_types::ir::{NonASAPOp, Operator}; use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; use asap_types::stages::Lifetime; -use asap_types::workload::{QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand}; +use asap_types::workload::{ + greatest_common_divisor, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand, +}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; /// How a [`ScheduleGroup`]'s evaluations add to the others'. @@ -283,10 +287,14 @@ pub fn executions_per_second<'e>( /// What computes a not-materialized node once per its own execution /// ([`consumers`]). -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Consumer { /// Query `.0` (an operator or scalar root), at its own rate. Query(usize), + /// Query `query` below a `TimeShift` anchored with `@ end()` or (when + /// `at_start`) `@ start()`: once per request, on the grid of its + /// request end (start) times ([`request_time_grid_ms`]). + QueryRequest { query: usize, at_start: bool }, /// A materialized node, at that node's rate. Materialized(PhysicalASAPNodeId), } @@ -310,6 +318,21 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap match shift.at { + Some(AtModifier::End) => Some(false), + Some(AtModifier::Start) => Some(true), + _ => None, + }, + _ => None, + }) + }; let walks = query_starts(dag) .into_iter() .enumerate() @@ -322,18 +345,46 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap> = HashMap::new(); for (consumer, start) in walks { let mut seen = HashSet::new(); - let mut stack: Vec = start; - while let Some(id) = stack.pop() { - if is_materialized.contains(&id) || !seen.insert(id) { + let mut stack: Vec<(PhysicalASAPNodeId, Consumer)> = + start.into_iter().map(|id| (id, consumer)).collect(); + while let Some((id, consumer)) = stack.pop() { + if is_materialized.contains(&id) || !seen.insert((id, consumer)) { continue; } consumers.entry(id).or_default().push(consumer); - stack.extend(inputs(id)); + // A query's input of an `@ end()` / `@ start()` shift runs once + // per request. + let below = match (consumer, request_anchor(id)) { + (Consumer::Query(query), Some(at_start)) => { + Consumer::QueryRequest { query, at_start } + } + (consumer, _) => consumer, + }; + stack.extend(inputs(id).map(|input| (input, below))); } } consumers } +/// The grid, in ms, of the times the requests of a query with `recurrence` +/// and `range_evaluation` anchor `@ end()` or `@ start()` (`at`) at: its +/// request end times, shifted back by its range-evaluation span for +/// `@ start()`; 1 ms when not known. +/// +/// Example: a query every 60 s over the last hour: `@ end()` 60,000 ms, +/// `@ start()` gcd(60,000, 3,600,000) = 60,000 ms. +pub fn request_time_grid_ms( + recurrence: &QueryRecurrence, + range_evaluation: Option, + at: AtModifier, +) -> u64 { + let end_ms = recurrence.evaluation_grid_ms(None).unwrap_or(1); + match (at, range_evaluation) { + (AtModifier::Start, Some(range)) => greatest_common_divisor(end_ms, range.span_ms), + _ => end_ms, + } +} + /// How often each node of `dag` is computed: `(multiplier, per_second)` /// (module doc, "Cost model"). A materialized node is computed once for all /// its queries: once per second at ingestion time (materialized at ingestion time), at @@ -342,12 +393,19 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap = HashMap::new(); for (id, of_node) in consumers(dag) { let entry = count.entry(id).or_default(); + // The queries reaching it, by evaluation grid: each evaluated at + // its steps, or once per request (below an `@ end()` / `@ start()` + // shift) when it reaches the node only so on that grid. + let mut queries_by_grid: BTreeMap> = BTreeMap::new(); for consumer in of_node { + match consumer { + Consumer::Query(query) => { + queries_by_grid + .entry(grid_ms(query)) + .or_default() + .insert(query, false); + } + Consumer::QueryRequest { query, at_start } => { + let at = match at_start { + true => AtModifier::Start, + false => AtModifier::End, + }; + let grid = entries.get(query).map_or(1, |entry| { + request_time_grid_ms(&entry.recurrence, entry.range_evaluation, at) + }); + queries_by_grid + .entry(grid) + .or_default() + .entry(query) + .or_insert(true); + } + Consumer::Materialized(node) => { + entry.0 += 1; + entry.1 += rate_of_materialized[&node]; + } + } + } + for queries in queries_by_grid.into_values() { entry.0 += 1; - entry.1 += match consumer { - Consumer::Query(query) => query_rates[query], - Consumer::Materialized(node) => rate_of_materialized[&node], - }; + entry.1 += evaluation_rate( + queries.iter().filter_map(|(&query, &per_request)| { + let entry = entries.get(query)?; + let range = (!per_request).then_some(entry.range_evaluation).flatten(); + Some((&entry.recurrence, range)) + }), + non_repeating_query_horizon_seconds, + )?; } } Ok(dag @@ -436,187 +528,4 @@ pub fn computations( } #[cfg(test)] -mod tests { - use super::*; - use asap_types::workload::{DemandEstimate, Rate, RepetitionInterval, TimestampMs}; - - fn fixed(interval: u32, phase: u64) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval), - evaluation_phase: TimestampMs(phase), - }) - } - - fn close(actual: f64, expected: f64) { - assert!((actual - expected).abs() < 1e-12, "{actual} != {expected}"); - } - - // Equal phases share evaluations; separate phases add. - #[test] - fn chosen_readers_preserve_phase_accounting() { - let queries = [ - fixed(20_000, 5_000), - fixed(20_000, 25_000), - fixed(20_000, 0), - ]; - close( - evaluation_rate(queries[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), - 1.0 / 20.0, - ); - close( - evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), - 2.0 / 20.0, - ); - // No global subset cache: each caller's selected queries determine the rate. - close(evaluation_rate(std::iter::empty(), 3600.0).unwrap(), 0.0); - } - - // Range offsets union within one cadence; retained state instead uses requests. - #[test] - fn range_offsets_and_request_rates_stay_distinct() { - let recurrence = fixed(60_000, 0); - let ranges = [ - Some(RangeEvaluation { - step_ms: 60_000, - span_ms: 3_600_000, - }), - Some(RangeEvaluation { - step_ms: 30_000, - span_ms: 600_000, - }), - None, - ]; - close( - evaluation_rate(ranges.iter().map(|r| (&recurrence, *r)), 3600.0).unwrap(), - 71.0 / 60.0, - ); - close( - evaluation_rate(ranges.iter().map(|_| (&recurrence, None)), 3600.0).unwrap(), - 1.0 / 60.0, - ); - } - - // Different fixed intervals remain additive even if timestamps can coincide. - #[test] - fn different_intervals_preserve_existing_conservative_rate() { - let queries = [fixed(20_000, 0), fixed(30_000, 0)]; - close( - evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), - 1.0 / 20.0 + 1.0 / 30.0, - ); - } - - // Estimates add with range multiplicity; scheduled/one-off/unknown use a maximum. - #[test] - fn estimated_and_batched_nonrepeating_demand_keep_their_policy() { - let estimate = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(2.0), - observed_at: None, - valid_for: None, - })); - let scheduled = QueryRecurrence::Repeated(RepeatedDemand::Scheduled(vec![ - TimestampMs(0), - TimestampMs(10), - ])); - let once = QueryRecurrence::OneTime { - invocations: 4, - execute_at: None, - }; - let unknown = QueryRecurrence::Unknown; - let range = Some(RangeEvaluation { - step_ms: 1000, - span_ms: 2000, - }); - close( - evaluation_rate( - [ - (&estimate, range), - (&estimate, None), - (&scheduled, range), - (&once, None), - (&unknown, None), - ], - 100.0, - ) - .unwrap(), - 8.0 + 6.0 / 100.0, - ); - } - - #[test] - fn billion_offset_mixed_ranges_are_counted_arithmetically() { - let recurrence = fixed(1000, 0); - let rate = evaluation_rate( - [ - ( - &recurrence, - Some(RangeEvaluation { - span_ms: 1_000_000_000, - step_ms: 2, - }), - ), - ( - &recurrence, - Some(RangeEvaluation { - span_ms: 1_000_000_000, - step_ms: 3, - }), - ), - ], - 60.0, - ) - .unwrap(); - assert_eq!(rate, 666_666_668.0); - } - - #[test] - fn arithmetic_union_matches_small_explicit_offset_sets() { - for left_step in 1..=7 { - for right_step in 1..=7 { - for left_span in [0, 5, 17, 30] { - for right_span in [0, 9, 23, 30] { - let mut ranges = BTreeMap::new(); - for (step, span) in - [(left_step, left_span), (right_step, right_span), (11, 29)] - { - let longest = ranges.entry(step).or_insert(0u64); - *longest = (*longest).max(span); - } - let mut oracle = BTreeSet::new(); - for (&step, &span) in &ranges { - oracle.extend((0..=span / step).map(|k| k * step)); - } - assert_eq!( - distinct_instants_per_request(&ranges), - oracle.len() as u64, - "{ranges:?}" - ); - } - } - } - } - let large = 1u64 << 63; - assert_eq!( - offset_intersection_count(&[(large - 1, large), (large - 3, large)]), - 1 - ); - } - - // Invalid fixed intervals and estimated rates still produce the evaluator's error. - #[test] - fn invalid_recurrence_is_rejected() { - assert!(matches!( - evaluation_rate([(&fixed(0, 0), None)], 3600.0), - Err(AnalyticalCostError::InvalidRecurrence) - )); - let invalid = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(-1.0), - observed_at: None, - valid_for: None, - })); - assert!(matches!( - evaluation_rate([(&invalid, None)], 3600.0), - Err(AnalyticalCostError::InvalidRecurrence) - )); - } -} +mod tests; diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs new file mode 100644 index 000000000..9cc26d4c2 --- /dev/null +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -0,0 +1,317 @@ +use super::*; +use asap_types::workload::{DemandEstimate, Rate, RepetitionInterval, TimestampMs}; + +fn fixed(interval: u32, phase: u64) -> QueryRecurrence { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval), + evaluation_phase: TimestampMs(phase), + }) +} + +fn close(actual: f64, expected: f64) { + assert!((actual - expected).abs() < 1e-12, "{actual} != {expected}"); +} + +// Equal phases share evaluations; separate phases add. +#[test] +fn chosen_readers_preserve_phase_accounting() { + let queries = [ + fixed(20_000, 5_000), + fixed(20_000, 25_000), + fixed(20_000, 0), + ]; + close( + evaluation_rate(queries[..2].iter().map(|r| (r, None)), 3600.0).unwrap(), + 1.0 / 20.0, + ); + close( + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), + 2.0 / 20.0, + ); + // No global subset cache: each caller's selected queries determine the rate. + close(evaluation_rate(std::iter::empty(), 3600.0).unwrap(), 0.0); +} + +// Range offsets union within one cadence; retained state instead uses requests. +#[test] +fn range_offsets_and_request_rates_stay_distinct() { + let recurrence = fixed(60_000, 0); + let ranges = [ + Some(RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }), + Some(RangeEvaluation { + step_ms: 30_000, + span_ms: 600_000, + }), + None, + ]; + close( + evaluation_rate(ranges.iter().map(|r| (&recurrence, *r)), 3600.0).unwrap(), + 71.0 / 60.0, + ); + close( + evaluation_rate(ranges.iter().map(|_| (&recurrence, None)), 3600.0).unwrap(), + 1.0 / 60.0, + ); +} + +// Different fixed intervals remain additive even if timestamps can coincide. +#[test] +fn different_intervals_preserve_existing_conservative_rate() { + let queries = [fixed(20_000, 0), fixed(30_000, 0)]; + close( + evaluation_rate(queries.iter().map(|r| (r, None)), 3600.0).unwrap(), + 1.0 / 20.0 + 1.0 / 30.0, + ); +} + +// Estimates add with range multiplicity; scheduled/one-off/unknown use a maximum. +#[test] +fn estimated_and_batched_nonrepeating_demand_keep_their_policy() { + let estimate = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(2.0), + observed_at: None, + valid_for: None, + })); + let scheduled = QueryRecurrence::Repeated(RepeatedDemand::Scheduled(vec![ + TimestampMs(0), + TimestampMs(10), + ])); + let once = QueryRecurrence::OneTime { + invocations: 4, + execute_at: None, + }; + let unknown = QueryRecurrence::Unknown; + let range = Some(RangeEvaluation { + step_ms: 1000, + span_ms: 2000, + }); + close( + evaluation_rate( + [ + (&estimate, range), + (&estimate, None), + (&scheduled, range), + (&once, None), + (&unknown, None), + ], + 100.0, + ) + .unwrap(), + 8.0 + 6.0 / 100.0, + ); +} + +#[test] +fn billion_offset_mixed_ranges_are_counted_arithmetically() { + let recurrence = fixed(1000, 0); + let rate = evaluation_rate( + [ + ( + &recurrence, + Some(RangeEvaluation { + span_ms: 1_000_000_000, + step_ms: 2, + }), + ), + ( + &recurrence, + Some(RangeEvaluation { + span_ms: 1_000_000_000, + step_ms: 3, + }), + ), + ], + 60.0, + ) + .unwrap(); + assert_eq!(rate, 666_666_668.0); +} + +#[test] +fn arithmetic_union_matches_small_explicit_offset_sets() { + for left_step in 1..=7 { + for right_step in 1..=7 { + for left_span in [0, 5, 17, 30] { + for right_span in [0, 9, 23, 30] { + let mut ranges = BTreeMap::new(); + for (step, span) in [(left_step, left_span), (right_step, right_span), (11, 29)] + { + let longest = ranges.entry(step).or_insert(0u64); + *longest = (*longest).max(span); + } + let mut oracle = BTreeSet::new(); + for (&step, &span) in &ranges { + oracle.extend((0..=span / step).map(|k| k * step)); + } + assert_eq!( + distinct_instants_per_request(&ranges), + oracle.len() as u64, + "{ranges:?}" + ); + } + } + } + } + let large = 1u64 << 63; + assert_eq!( + offset_intersection_count(&[(large - 1, large), (large - 3, large)]), + 1 + ); +} + +/// One workload entry per recurrence, each with `range_evaluation`. +fn entries( + recurrence: &[QueryRecurrence], + range_evaluation: Option, +) -> Vec { + recurrence + .iter() + .map(|recurrence| QueryWorkloadEntry { + query: asap_types::workload::Query("q".into()), + requirements: Default::default(), + predictability: Default::default(), + recurrence: recurrence.clone(), + range_evaluation, + }) + .collect() +} + +/// A plan node `id` running `payload` with `timing` and `lifetime`. +fn node( + id: PhysicalASAPNodeId, + payload: Operator, + timing: ExecutionTiming, + lifetime: Lifetime, +) -> asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode { + use asap_types::ir::properties::{DataPrimitive, ExecutionDataState}; + use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; + asap_types::plan::physical_asap_dag::PhysicalASAPDAGNode { + id, + output_state: ExecutionDataState { + timing, + primitive: DataPrimitive::of(&payload, asap_types::ir::OperatorResultKind::RangeVector), + }, + parameters: matches!(payload, Operator::ASAP(_)) + .then_some(asap_types::summary_type::SketchParams::Kll { k: 200 }), + payload, + lifetime, + output_schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::new("value", FieldDataType::Plain(DataType::Float64), false), + ], + 0, + vec![], + ), + retention_ms: None, + selected_physical_candidate: None, + view_coverage: None, + } +} + +fn scan() -> Operator { + use asap_types::ir::operator::Source; + use asap_types::ir::schema::{DataType, Field, Schema}; + Operator::NonASAP(asap_types::ir::NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +fn last(seconds: u64, child: PhysicalASAPNodeId) -> Operator { + Operator::NonASAP(asap_types::ir::NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(seconds), + kind: asap_types::ir::TimeRangeKind::Range, + child, + }) +} + +/// `m[5m]` read by three queries: two every 10 s at phase 0, one every +/// 30 s. The two on the 10 s grid are one execution context, which +/// computes the not-materialized range and scan once for both: two +/// contexts, 1/10 + 1/30 per second. +#[test] +fn a_not_materialized_node_is_computed_once_per_evaluation_grid() { + let query_time = ExecutionTiming::QueryTime; + let dag = PhysicalASAPDAG::new( + vec![ + node(0, scan(), query_time, Lifetime::None), + node(1, last(300, 0), query_time, Lifetime::None), + ], + vec![1, 1, 1], + vec![], + ) + .unwrap(); + let recurrence = [fixed(10_000, 0), fixed(10_000, 0), fixed(30_000, 0)]; + let rates = computations(&dag, &entries(&recurrence, None), 3600.0).unwrap(); + for id in [0, 1] { + assert_eq!(rates[&id].0, 2); + close(rates[&id].1, 1.0 / 10.0 + 1.0 / 30.0); + } +} + +/// `max_over_time(m[5m] @ end())` requested every 60 s over the last +/// hour at a 10 s step (361 steps per request): the shift runs at +/// every step, its input (the range and the scan) once per request, +/// at the request end: 1/60 per second, not 361/60. +#[test] +fn a_range_query_computes_its_at_end_subtree_once_per_request() { + use asap_types::ir::operator::TimeShift; + let query_time = ExecutionTiming::QueryTime; + let at_end = Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(AtModifier::End), + }, + child: 1, + }); + let dag = PhysicalASAPDAG::new( + vec![ + node(0, scan(), query_time, Lifetime::None), + node(1, last(300, 0), query_time, Lifetime::None), + node(2, at_end, query_time, Lifetime::None), + ], + vec![2], + vec![], + ) + .unwrap(); + let hour = RangeEvaluation { + step_ms: 10_000, + span_ms: 3_600_000, + }; + let rates = computations(&dag, &entries(&[fixed(60_000, 0)], Some(hour)), 3600.0).unwrap(); + close(rates[&2].1, 361.0 / 60.0); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 1.0 / 60.0); + } +} + +// Invalid fixed intervals and estimated rates still produce the evaluator's error. +#[test] +fn invalid_recurrence_is_rejected() { + assert!(matches!( + evaluation_rate([(&fixed(0, 0), None)], 3600.0), + Err(AnalyticalCostError::InvalidRecurrence) + )); + let invalid = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(-1.0), + observed_at: None, + valid_for: None, + })); + assert!(matches!( + evaluation_rate([(&invalid, None)], 3600.0), + Err(AnalyticalCostError::InvalidRecurrence) + )); +} diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index 81e58ace6..dccb77b05 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -57,6 +57,16 @@ fn selected_plan_objective_and_plan_cost( let models = PlanningModels::builtin() .with_cost_inputs(settings) .with_deployment_capabilities(&capabilities); + selected_plan_objective_and_plan_cost_with(workload, models, settings) +} + +/// The plan Stage 3 selects for `workload` with `models`, whose cost +/// inputs are `settings`, with the MILP's optimum and its `plan_cost`. +fn selected_plan_objective_and_plan_cost_with( + workload: &PlanningWorkload, + models: PlanningModels<'_>, + settings: &CostInputs, +) -> (PhysicalASAPDAG, (f64, f64)) { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap(); let roots = lower_promql_roots(workload).unwrap(); @@ -230,3 +240,80 @@ fn raw_data_on_disk_costing_less_than_the_solver_tolerance_still_steers_selectio "selected {plan_cost} is not cheaper than {ignoring_disk_cost}, the plan selected ignoring disk" ); } + +/// Two windowed quantiles over one metric (1 m and 5 m, every 10 s; 10,000 +/// series every 15 s), accuracy not enforced: the selected plan shares one +/// query-time scan, whose execution reads the 5 m window once and retains +/// it ($h_v$, constraint (R)), at most 0.144/s. +#[test] +fn the_objective_is_the_plan_cost_on_a_shared_scan() { + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [ + every_10s("quantile_over_time(0.5, data[1m])"), + every_10s("quantile_over_time(0.9, data[5m])"), + ], + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 666.67}}, + "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}, + }, + })) + .unwrap(); + let models = PlanningModels::builtin().with_ignore_accuracy_guarantees(true); + let (_, (objective, plan_cost)) = + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()); + assert_equal("shared scan", (objective, plan_cost)); + assert!(plan_cost <= 0.144, "{plan_cost}"); +} + +/// The o11y benchmark's q21: `sum(rate(..[5m] offset 1h))`, a 1 h range +/// query every 60 s. Raw rows it reads may be materialized at ingestion +/// time behind the offset (a `TimeShift` over the `Scan`); that store +/// keeps 2 h 5 m of rows (its retention: the 1 h span, the 5 m range and +/// the 1 h offset), which its `TimeRange` reads 5 m of, in the MILP and in +/// `plan_cost` alike (the MILP's objective was 0.326/s for a plan +/// `plan_cost` charged 0.0219/s, one second of rows retained). +#[test] +fn the_objective_is_the_plan_cost_on_an_offset_range_query() { + let source = serde_json::json!({"sample_interval": 1000, "series_count": 1000, "metric_type": "counter"}); + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": { + "language": "promql", + "query_batch": null, + "repeating_queries": [{ + "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m] offset 1h))", + "demand": {"fixed_interval_at": {"interval": 60000, "evaluation_phase": 0}}, + "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, + "predictability": {"predictable": {"known_at": null}}, + "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}, + }], + }, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": {"http_requests_total": source}, + }, + })) + .unwrap(); + assert_equal( + "offset range query", + selected_plan_objective_and_plan_cost_with( + &w, + PlanningModels::builtin(), + &CostInputs::default(), + ) + .1, + ); +} + +/// A query `query` every 10 s at phase 0, within 1 % at δ = 0.01. +fn every_10s(query: &str) -> serde_json::Value { + serde_json::json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": 10000, "evaluation_phase": 0}}, + "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.01}}, + "predictability": {"predictable": {"known_at": null}}, + }) +} diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index ef570d680..c4a2ca803 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -22,6 +22,7 @@ use asap_planner::{ SelectedPlanReport, UserInput, }; use asap_types::ir::operator::Source; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; use futures::{executor::block_on, StreamExt}; @@ -34,11 +35,19 @@ const EVALUATION_MS: i64 = 3_600_000; /// δ 0.01, over 1,000 ingesting `latency_ms` series sampled every second: /// #509 Example 4b when the one query is its p99. fn workload(queries: &[&str]) -> PlanningWorkload { + let every_10_minutes: Vec<(&str, u64)> = + queries.iter().map(|&query| (query, 600_000)).collect(); + workload_every(&every_10_minutes) +} + +/// [`workload`] with each query repeating every `interval_ms` of its own, at +/// phase 0. +fn workload_every(queries: &[(&str, u64)]) -> PlanningWorkload { let entries: Vec = queries .iter() - .map(|query| { + .map(|(query, interval_ms)| { format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": 600000, "evaluation_phase": 0}}}}, + r#"{{"query": {query:?}, "demand": {{"fixed_interval_at": {{"interval": {interval_ms}, "evaluation_phase": 0}}}}, "requirements": {{"accuracy": {{"accuracy_metric": "rank", "max_error": 0.01, "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, "max_latency_ms": 200.0}}, @@ -248,6 +257,53 @@ async fn the_plan_carries_stage3s_predicted_usage() { assert!(usage.per_node.iter().any(|node| node.node_id == root)); } +/// Example 4b's p99 every 10 minutes and its p50 every 30 minutes: two +/// evaluation contexts. The chain of windows maintained at ingestion time +/// runs once, whoever reads it, and retains the hour both read back +/// ($h_v$, constraint (R)); each context reads it with query-time nodes of +/// its own. +#[tokio::test] +async fn one_chain_serves_two_evaluation_contexts() { + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..DeploymentCapabilities::default() + }; + let (plan, _) = plan_with_report_for( + workload_every(&[ + ("quantile_over_time(0.99, latency_ms[60m])", 600_000), + ("quantile_over_time(0.5, latency_ms[60m])", 1_800_000), + ]), + &capabilities, + ) + .await; + let chains: Vec<_> = plan + .dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + }) + .collect(); + assert_eq!(chains.len(), 1, "{:#?}", plan.dag.nodes); + assert_eq!(chains[0].retention_ms, Some(3_600_000)); + let ranges_over_the_chain = plan + .dag + .nodes + .iter() + .filter(|node| { + matches!( + node.payload, + Operator::NonASAP(NonASAPOp::TimeRange { child, .. }) if child == chains[0].id + ) + }) + .count(); + assert_eq!(ranges_over_the_chain, 2); + assert_ne!(plan.dag.roots[0], plan.dag.roots[1]); +} + /// `sum(latency_ms)` reads an instant selector: per series, only the latest /// sample in its 1 s lookback counts. Series `a` has samples at T-900 ms (1) /// and T-100 ms (10), series `b` at T-200 ms (100): the sum is 10 + 100. diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 40f0b350c..90f9bd3d5 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -1,121 +1,183 @@ -//! The baseline objective's coefficients +//! The objective's coefficients //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 3, "Stage -//! 3: baseline selection and plan output"): $w_v$ of every physical -//! candidate and $c_{u,c,i}$ of every edge a plan may select, from -//! `asap-cost` under cost model 1. +//! sections 3 and 5, "Cost and retention") under cost model 1, from +//! `asap-cost`: per execution of a candidate in a context +//! ([`crate::selectable`]), per edge in a context, and per amount a +//! candidate retains: //! -//! Each candidate is costed as one plan node ([`candidate_node`], -//! [`asap_cost::node_resource_usage_and_cost`]) over the logical output of each input: the -//! most any costed candidate it may read there (`Memo::producers`) -//! outputs (rows, bytes and, for a state, its family). That is $w_v$, which -//! does not depend on the producers chosen. The part of a consumer's cost -//! that changes with the producer read is on the edge: $c_{u,c,i}$ is the -//! consumer's cost with input $i$ read from $u$'s own output, minus $w_c$. -//! It is exact when a consumer's cost changes with one input's producer -//! only (every consumer of a state reads one state input). Every state -//! candidate has its own θ (a merge, a `TimeRange` over a chain carry the -//! θ of the states they read, and read only producers with it), so a -//! consumer above a sketch build is costed at that build's θ. +//! $$\sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} +//! \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum_{\kappa,u,c,i} +//! \rho_\kappa c_{\kappa,u,c,i} y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s +//! r_s,$$ //! -//! How often a candidate runs is the baseline's fixed rate: at ingestion -//! time, one second of arriving rows per second; at query time, once per -//! evaluation of the queries reaching its group (once per request when it -//! is materialized across evaluations). A materialized candidate and a -//! query-time `Scan` retain the most their selectable consumers read back -//! ([`retention_need_ms`]). Execution contexts make both per context. +//! plus the request overhead every plan pays. Each candidate is costed as +//! one plan node ([`candidate_node`], +//! [`asap_cost::node_resource_usage_and_cost`]) over a baseline of each +//! input: the most any producer it may read (on its admissible edges) +//! outputs (rows, bytes and, for a state, its θ), at its execution's rate +//! $\rho_\kappa$ ([`executions_per_second`]). That does not depend on the +//! producers chosen. An ingestion-time consumer reads one batch of raw +//! data, also from a `Scan` materialized at ingestion time +//! ([`reads_one_batch_of`]). The part of a consumer's cost that changes +//! with the producer read is on the edge: $\rho_\kappa c_{\kappa,u,c,i}$ is +//! the consumer's cost in $\kappa$ with input $i$ read from $u$'s own +//! output, minus its own. It is exact when a consumer's cost changes with +//! one input's producer only (every consumer of a state reads one state +//! input). Every state candidate has its own θ (a merge, a `TimeRange` over +//! a chain carry the θ of the states they read, and may read only +//! producers with it), so the baseline over its admissible producers is +//! that θ's. +//! +//! An execution of a query-time `Scan` reads, once per run, the most any +//! consumer it may have in its context reads back in one run +//! ([`edge_scan_read_ms`]), a constant of the execution: exact when that +//! consumer is selected. +//! +//! A candidate that retains time-indexed rows +//! (`PhysicalASAPDAGNode::retains_rows`) is costed retaining the most its +//! consumers read back ([`edge_retention_need_ms`]), the largest $h_v$ (R) +//! allows. Retention is linear in that amount: $\pi_v$ is what retaining +//! one more window (or second) costs there, and $W^{\mathrm{mat}}_v$ of a +//! materialized candidate is its cost less $\pi_v h_v$. A query-time +//! scan's retention is raw data on disk, kept once per source for the +//! selected scan that keeps the most: $r_s \ge \pi_v h_v$ +//! ([`asap_cost::raw_data_disk_cost_per_second`]). A candidate one of whose +//! consumers reads it unbounded (a `Scan` read whole, a `TimeShift` +//! anchored with `@`) has no $h_v$: it is costed retaining everything. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; -use asap_cost::plan::execution_demand::executions_per_second; +use asap_cost::plan::execution_demand::{evaluation_rate, executions_per_second as rate_of}; use asap_cost::{ maintained_window_ms, node_resource_usage_and_cost, raw_data_disk_cost_per_second, CostInputs, NodeResourceUsageAndCost, ProducerOutput, }; -use asap_types::ir::operator::Source; +use asap_types::ir::operator::{AtModifier, Source}; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::{NonASAPOp, Operator}; use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, }; use asap_types::stages::{ - Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, + ExecutionContext, Lifetime, Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, }; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; +use crate::milp::model::QueryId; use crate::plan::candidate_node; -use crate::retention::{retention_need_ms, scan_read_ms}; +use crate::retention::{edge_retention_need_ms, edge_scan_read_ms, output_retention_need_ms}; +use crate::selectable::{evaluation_grid, request_time_grid_ms, Execution, ExecutionContexts}; -/// $w_v$ and $c_{u,c,i}$. -pub(crate) struct SelectionCosts { - /// $w_v$, cost per second, by [`PhysicalCandidateId`], or why - /// `asap-cost` could not cost it (the MILP leaves such a candidate out). - pub(crate) candidate_cost_per_second: Vec>, - /// $c_{u,c,i}$, the change of its consumer's cost per second, of each - /// selectable edge whose producer and consumer are costable; an edge - /// missing here cannot be selected. - pub(crate) edge_cost_change_per_second: HashMap, - /// By [`PhysicalCandidateId`], for a query-time `Scan`: its source and - /// the cost per second of keeping the raw data it reads on disk - /// ([`asap_cost::raw_data_disk_cost_per_second`]). A plan keeps each - /// source once, for the selected scan that keeps the most, as - /// `plan_cost` charges it. - pub(crate) raw_data_on_disk: Vec>, +/// The objective's coefficients +/// (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +/// section 5, "Cost and retention"), per second. +pub(crate) struct ExecutionCosts { + /// $\rho_\kappa c_{\kappa,v}$ of each execution, $W^{\mathrm{mat}}_v$ + /// of an execution in `Materialized(v)`, by position in + /// `ExecutionContexts::executions`; or why `asap-cost` could not cost + /// it (the MILP leaves it out). + pub(crate) per_execution: Vec>, + /// $\rho_\kappa c_{\kappa,u,c,i}$ of each edge, by position in + /// `ExecutionContexts::edges`; `None` when it could not be costed (the + /// MILP leaves it out). + pub(crate) per_edge: Vec>, + /// $\pi_v$, per ms retained, of each candidate that retains rows + /// (`PhysicalASAPDAGNode::retains_rows`) only within a bound: it has + /// $h_v$ and constraint (R). One whose consumer reads it unbounded is + /// costed retaining everything. + pub(crate) retention_cost_per_second: HashMap, + /// The source of each query-time `Scan`: its retention is raw data on + /// disk, which a plan keeps once per source for the selected scan that + /// keeps the most, as `plan_cost` charges it ($r_s$). + pub(crate) raw_data_source: HashMap, + /// The cost per second of the raw data a query-time `Scan` without + /// $h_v$ keeps on disk, retaining everything. + pub(crate) unbounded_raw_data_on_disk_cost_per_second: HashMap, /// The runtime's fixed overhead per query request, per second /// ([`asap_cost::request_overhead_cost_per_second`]): every plan pays /// it. pub(crate) request_overhead_cost_per_second: f64, } -/// [`SelectionCosts`] over `selectable` (each admissible edge a plan may -/// select, by consumer and input). Errs on nothing: a candidate -/// `asap-cost` cannot cost is reported in -/// [`SelectionCosts::candidate_cost_per_second`]. -pub(crate) fn selection_costs( +/// The objective's coefficients of every execution and edge of `contexts` +/// ([`ExecutionCosts`]; module doc). `query_outputs` are the executions +/// that may answer each query output. Errs on nothing: what `asap-cost` +/// cannot cost is reported in [`ExecutionCosts::per_execution`] or left +/// out of [`ExecutionCosts::per_edge`]. +pub(crate) fn execution_costs( physical: &PhysicalASAPMemoCandidates, - selectable: &[PhysicalEdgeId], + contexts: &ExecutionContexts, + query_outputs: &[(QueryId, Vec)], entries: &[QueryWorkloadEntry], data: &DataWorkload, cost_inputs: &CostInputs, request_overhead_cost_per_second: f64, -) -> SelectionCosts { +) -> ExecutionCosts { let memo = &physical.logical_candidates.memo; let count = physical.physical_candidates.len(); - let mut out_edges: Vec> = vec![Vec::new(); count]; - for &edge in selectable { - out_edges[physical.edge(edge).producer_candidate_id.0].push(edge); - } - // The candidates that may read each candidate. - let readers_of = |v: PhysicalCandidateId| { - out_edges[v.0] - .iter() - .map(|&edge| physical.edge(edge).consumer_candidate_id) - }; // Each candidate's node, its inputs numbered 0.. (the ids do not - // matter to `node_resource_usage_and_cost`), with the retention it would need. + // matter to `node_resource_usage_and_cost`), retaining nothing yet. + let node_retaining = |v: PhysicalCandidateId, retention_ms: Option| { + let inputs: Vec = (0..physical.input_count(v)).collect(); + let mut node = candidate_node(physical, v, v.0, &inputs); + node.retention_ms = retention_ms; + node + }; + let retains_rows: Vec = (0..count) + .map(|v| node_retaining(PhysicalCandidateId(v), None).retains_rows()) + .collect(); + let query_time_scan: Vec = (0..count) + .map(|v| is_query_time_scan(physical, PhysicalCandidateId(v))) + .collect(); + // The most any consumer of each candidate that retains rows reads back + // ($h_v$ at its largest): `None` when one reads it unbounded. + let mut most_read_ms: HashMap> = (0..count) + .filter(|&v| retains_rows[v]) + .map(|v| (PhysicalCandidateId(v), Some(0))) + .collect(); + let mut read = |v: PhysicalCandidateId, need_ms: Option| { + if let Some(most) = most_read_ms.get_mut(&v) { + *most = most.zip(need_ms).map(|(most, need)| most.max(need)); + } + }; + for &(_, edge, producer) in contexts.edges() { + read( + producer.candidate, + edge_retention_need_ms(physical, entries, edge), + ); + } + for (query, executions) in query_outputs { + for execution in executions { + read( + execution.candidate, + Some(output_retention_need_ms(&entries[query.0])), + ); + } + } + // Each query-time `Scan`'s read per execution at its largest, and per + // execution: the most any consumer it may have in that context reads + // in one run (`None`: one reads it whole), charged once. + let mut scan_read_ms: HashMap> = (0..count) + .filter(|&v| query_time_scan[v]) + .map(|v| (PhysicalCandidateId(v), Some(0))) + .collect(); + let mut execution_read_ms: Vec>> = vec![None; contexts.executions().len()]; + for &(_, edge, producer) in contexts.edges() { + let Some(most) = scan_read_ms.get_mut(&producer.candidate) else { + continue; + }; + let read_ms = edge_scan_read_ms(physical, edge); + *most = most.zip(read_ms).map(|(most, read)| most.max(read)); + let of_execution = &mut execution_read_ms[contexts.index_of(producer)]; + *of_execution = Some(match *of_execution { + None => read_ms, + Some(most) => most.zip(read_ms).map(|(most, read)| most.max(read)), + }); + } + let scan_read_of = |v: PhysicalCandidateId| scan_read_ms.get(&v).copied().flatten(); let nodes: Vec = (0..count) .map(PhysicalCandidateId) - .map(|v| { - let inputs: Vec = (0..physical.input_count(v)).collect(); - let mut node = candidate_node(physical, v, v.0, &inputs); - if node.retains_rows() { - node.retention_ms = retention_need_ms(physical, entries, v, readers_of); - } - node - }) - .collect(); - // Each query-time `Scan`'s read per execution: the most any selectable - // consumer reads (no range-evaluation span), charged once. - let scan_read_ms_of: Vec> = nodes - .iter() - .enumerate() - .map(|(v, node)| { - let query_time_scan = node.output_state.timing == ExecutionTiming::QueryTime - && matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. })); - query_time_scan - .then(|| scan_read_ms(physical, PhysicalCandidateId(v), readers_of)) - .flatten() - }) + .map(|v| node_retaining(v, most_read_ms.get(&v).copied().flatten())) .collect(); // Each group's expression as a payload, for its logical output. let group_payloads: Vec = memo @@ -132,123 +194,330 @@ pub(crate) fn selection_costs( let order = memo .topological_order() .unwrap_or_else(|| (0..memo.groups().len() as u32).map(MemoGroupId).collect()); + // A candidate's (own, raw data on disk) cost per second as `node` over + // `inputs`, a query-time `Scan` reading `scan_read_ms` per execution. + let cost_reading = |v: PhysicalCandidateId, + node: &PhysicalASAPDAGNode, + inputs: &[ProducerOutput<'_>], + per_second: f64, + scan_read_ms: Option| { + cost_node( + physical, + v, + node, + inputs, + per_second, + scan_read_ms, + data, + cost_inputs, + ) + .and_then(|usage| cost_per_second(&usage, cost_inputs)) + }; + let cost = + |v: PhysicalCandidateId, + node: &PhysicalASAPDAGNode, + inputs: &[ProducerOutput<'_>], + per_second: f64| { cost_reading(v, node, inputs, per_second, scan_read_of(v)) }; + // Each candidate's usage of one computation (what it outputs does not + // depend on how often it runs) over its baseline inputs. let mut usage: Vec, String>>> = (0..count).map(|_| None).collect(); - // Each candidate's logical inputs, as costed for $w_v$; each edge into - // it replaces one. - let mut inputs_of: Vec>>> = (0..count).map(|_| None).collect(); + let mut baseline: Vec>>> = vec![None; count]; for group in order { for &v in &candidates_of_group[group.0 as usize] { - let inputs = logical_inputs(physical, &group_payloads, &candidates_of_group, &usage, v); - usage[v.0] = Some(match &inputs { + baseline[v.0] = baseline_inputs(memo, physical, &group_payloads, &nodes, &usage, v); + usage[v.0] = Some(match &baseline[v.0] { Some(inputs) => cost_node( physical, + v, &nodes[v.0], inputs, - entries, + 1.0, + scan_read_of(v), data, cost_inputs, - scan_read_ms_of[v.0], ), - None => Err("an input group has no costable candidate".to_string()), + None => Err("an input has no costable producer".to_string()), }); - inputs_of[v.0] = inputs; } } - let candidate_cost_per_second: Vec> = usage + // Each execution's (own, raw data on disk) cost per second at its + // rate, over its baseline inputs, and that rate. + let at_rate: Vec> = contexts + .executions() .iter() - .map(|usage| match usage { - Some(Ok(usage)) => cost_per_second(usage), - Some(Err(error)) => Err(error.clone()), - None => Err("it reads a group on a cycle of candidates".to_string()), + .zip(&execution_read_ms) + .map(|(&execution, read_ms)| { + let v = execution.candidate; + if let Some(Err(error)) = &usage[v.0] { + return Err(error.clone()); + } + let per_second = executions_per_second( + physical, + entries, + contexts, + execution, + cost_inputs.non_repeating_query_horizon_seconds, + )?; + let inputs = baseline[v.0] + .as_ref() + .ok_or("it reads a group on a cycle of candidates")?; + let read_ms = read_ms.unwrap_or_else(|| scan_read_of(v)); + let (own, raw_data) = cost_reading(v, &nodes[v.0], inputs, per_second, read_ms)?; + Ok((own, raw_data, per_second)) }) .collect(); - let raw_data_on_disk = usage + // The one execution of each materialized candidate, by position. + let own_execution: HashMap = contexts + .executions() .iter() - .map(|usage| { - let (source, bytes) = usage.as_ref()?.as_ref().ok()?.raw_scan?; - let cost = raw_data_disk_cost_per_second(bytes, cost_inputs)?; - Some((source.clone(), cost)) - }) + .enumerate() + .filter(|(_, execution)| materialized(physical, execution.candidate)) + .map(|(index, execution)| (execution.candidate, index)) .collect(); - let mut edge_cost_change_per_second = HashMap::new(); - for &id in selectable { - let e = physical.edge(id); - let (u, c) = (e.producer_candidate_id, e.consumer_candidate_id); - let (Some(Ok(produced)), Ok(own)) = (&usage[u.0], &candidate_cost_per_second[c.0]) else { + // π_v of a candidate retaining rows every consumer reads within a bound: + // what one more Δ ms of retention costs, over Δ (a window of a chain, + // else a second), at its retention's largest; its whole own cost when + // it is materialized, its raw data on disk for a query-time scan (each + // of its runs reads what it reads at that retention, ρ_κ c_{κ,v}). + let mut retention_cost_per_second = HashMap::new(); + for (&v, &most_read_ms) in &most_read_ms { + let Some(retention_ms) = most_read_ms else { continue; }; - let Some(mut inputs) = inputs_of[c.0].clone() else { + let delta_ms = physical.physical_candidates[v.0] + .chain_tumbling_window_width_ms(physical) + .unwrap_or(1_000) + .max(1); + let longer = node_retaining(v, Some(retention_ms + delta_ms)); + let per_ms = match materialized(physical, v) { + true => own_execution.get(&v).and_then(|&index| { + let (own, raw_data, per_second) = at_rate[index].as_ref().ok()?; + let inputs = baseline[v.0].as_ref()?; + let (longer_own, longer_raw_data) = cost(v, &longer, inputs, *per_second).ok()?; + Some((longer_own + longer_raw_data - own - raw_data) / delta_ms as f64) + }), + false => usage[v.0] + .as_ref() + .and_then(|usage| usage.as_ref().ok()) + .and_then(|_| { + let inputs = baseline[v.0].as_ref()?; + let (_, raw_data) = cost(v, &nodes[v.0], inputs, 1.0).ok()?; + let (_, longer_raw_data) = cost(v, &longer, inputs, 1.0).ok()?; + Some((longer_raw_data - raw_data) / delta_ms as f64) + }), + }; + if let Some(per_ms) = per_ms { + retention_cost_per_second.insert(v, per_ms.max(0.0)); + } + } + // Each query-time scan's source and, without $h_v$, the raw data it + // keeps on disk retaining everything. + let mut raw_data_source = HashMap::new(); + let mut unbounded_raw_data_on_disk_cost_per_second = HashMap::new(); + for v in (0..count) + .filter(|&v| query_time_scan[v]) + .map(PhysicalCandidateId) + { + let Some(Ok(scan)) = &usage[v.0] else { continue; }; - inputs[e.input_index_of_consumer] = - produced.producer_output(&nodes[u.0].payload, &nodes[u.0].output_schema); - let with_u = cost_node( - physical, - &nodes[c.0], - &inputs, - entries, - data, - cost_inputs, - scan_read_ms_of[c.0], - ) - .and_then(|usage| cost_per_second(&usage)); - if let Ok(with_u) = with_u { - edge_cost_change_per_second.insert(id, with_u - own); + let Some((source, _)) = scan.raw_scan else { + continue; + }; + raw_data_source.insert(v, source.clone()); + if !retention_cost_per_second.contains_key(&v) { + if let Some(Ok((_, raw_data))) = baseline[v.0] + .as_ref() + .map(|inputs| cost(v, &nodes[v.0], inputs, 1.0)) + { + unbounded_raw_data_on_disk_cost_per_second.insert(v, raw_data); + } } } - SelectionCosts { - candidate_cost_per_second, - edge_cost_change_per_second, - raw_data_on_disk, + let per_execution: Vec> = contexts + .executions() + .iter() + .zip(&at_rate) + .map(|(execution, at_rate)| { + let &(own, raw_data, _) = at_rate.as_ref().map_err(Clone::clone)?; + let v = execution.candidate; + Ok(match retention_cost_per_second.get(&v) { + // A query-time scan's raw data is charged per source. + _ if query_time_scan[v.0] => own, + Some(per_ms) if materialized(physical, v) => { + let retention_ms = most_read_ms[&v].unwrap_or(0) as f64; + own + raw_data - per_ms * retention_ms + } + _ => own + raw_data, + }) + }) + .collect(); + let per_edge = contexts + .edges() + .iter() + .map(|&(consumer, edge, producer)| { + let e = physical.edge(edge); + let (u, c) = (e.producer_candidate_id, e.consumer_candidate_id); + per_execution[contexts.index_of(producer)].as_ref().ok()?; + let &(own, raw_data, per_second) = + at_rate[contexts.index_of(consumer)].as_ref().ok()?; + let produced = usage[u.0].as_ref()?.as_ref().ok()?; + let mut inputs = baseline[c.0].clone()?; + if !reads_one_batch_of(&nodes, c, u) { + inputs[e.input_index_of_consumer] = + produced.producer_output(&nodes[u.0].payload, &nodes[u.0].output_schema); + } + let (with_u_own, with_u_raw_data) = cost(c, &nodes[c.0], &inputs, per_second).ok()?; + Some(with_u_own + with_u_raw_data - own - raw_data) + }) + .collect(); + ExecutionCosts { + per_execution, + per_edge, + retention_cost_per_second, + raw_data_source, + unbounded_raw_data_on_disk_cost_per_second, request_overhead_cost_per_second, } } -/// The logical output of each input of `v` over the costed candidates -/// (`usage`) it may read there (`Memo::producers`); `None` when one has -/// none. -fn logical_inputs<'a>( - physical: &'a PhysicalASAPMemoCandidates, +/// Whether candidate `v` is materialized across evaluations. +fn materialized(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { + matches!( + physical.physical_candidates[v.0] + .physical_properties + .lifetime(), + Lifetime::AcrossEvaluations(_) + ) +} + +/// Whether candidate `v` is a `Scan` at query time, which reads raw data +/// in each execution. +fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { + let candidate = &physical.physical_candidates[v.0]; + candidate.physical_properties.timing() == ExecutionTiming::QueryTime + && matches!( + physical + .logical_candidates + .memo + .candidate(candidate.logical_candidate_id) + .expression + .operator, + Operator::NonASAP(NonASAPOp::Scan { .. }) + ) +} + +/// $\rho$ of `execution`'s context, how often it runs, from the queries it +/// serves: an ingestion-time execution once per second of arriving rows; +/// in an evaluation context, the combined evaluation rate +/// ([`evaluation_rate`]) of the queries on its grid that reach its group, +/// with their range evaluations, and of the range queries whose request +/// end (start) times are the grid, once per request (an input of a +/// `TimeShift` anchored with `@`); in `Materialized(m)`, as often as $m$ +/// runs ([`asap_cost::plan::execution_demand::executions_per_second`]). +/// +/// Example: queries every 10 s, 10 s and 30 s reaching a scan: 1/10 per +/// second in the 10 s context, 1/30 in the 30 s one. +pub(crate) fn executions_per_second( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + contexts: &ExecutionContexts, + execution: Execution, + non_repeating_query_horizon_seconds: f64, +) -> Result { + let candidate = &physical.physical_candidates[execution.candidate.0]; + if candidate.physical_properties.timing() == ExecutionTiming::IngestionTime { + return Ok(1.0); + } + let memo = &physical.logical_candidates.memo; + let reaching = |v: PhysicalCandidateId| { + let group = physical.physical_candidates[v.0].logical_candidate_id.group; + memo.groups()[group.0 as usize] + .reached_by_queries() + .iter() + .filter_map(|&query| entries.get(query)) + }; + let rate = match contexts.context(execution.context) { + ExecutionContext::Evaluation { grid } => evaluation_rate( + reaching(execution.candidate).filter_map(|entry| { + if evaluation_grid(entry) == grid { + return Some((&entry.recurrence, entry.range_evaluation)); + } + let request_time = [AtModifier::End, AtModifier::Start] + .into_iter() + .any(|at| grid.period_ms == request_time_grid_ms(entry, at)); + (entry.range_evaluation.is_some() && request_time) + .then_some((&entry.recurrence, None)) + }), + non_repeating_query_horizon_seconds, + ), + ExecutionContext::Materialized(m) => { + let properties = physical.physical_candidates[m.0].physical_properties; + rate_of( + properties.timing(), + properties.lifetime(), + reaching(m), + non_repeating_query_horizon_seconds, + ) + } + // Only ingestion-time candidates run in an ingestion pipeline. + ExecutionContext::IngestionPipeline(_) => Ok(1.0), + }; + rate.map_err(|error| error.to_string()) +} + +/// Candidate `v`'s baseline inputs, by input index: over the producers it +/// may read (its admissible edges) whose usage is known, the most any +/// outputs ([`logical_output`]); `None` when an input has none. A producer +/// of which `v` reads one batch ([`reads_one_batch_of`]) is left out. +fn baseline_inputs<'a>( + memo: &'a Memo, + physical: &PhysicalASAPMemoCandidates, group_payloads: &'a [PhysicalASAPOperatorPayload], - candidates_of_group: &[Vec], + nodes: &[PhysicalASAPDAGNode], usage: &[Option, String>>], v: PhysicalCandidateId, ) -> Option>> { - let memo = &physical.logical_candidates.memo; - let consumer = physical.physical_candidates[v.0].logical_candidate_id; - input_groups(memo, physical, v) + memo.candidate(physical.physical_candidates[v.0].logical_candidate_id) + .expression + .operator + .children() .into_iter() .enumerate() - .map(|(input, group)| { - let readable: HashSet<_> = memo.producers(consumer, input).into_iter().collect(); + .map(|(i, input)| { logical_output( - &group_payloads[group.0 as usize], - &memo.groups()[group.0 as usize].group_properties.schema, - candidates_of_group[group.0 as usize] + &group_payloads[input.0 as usize], + &memo.groups()[input.0 as usize].group_properties.schema, + physical + .admissible_edges_into(v, i) .iter() - .filter(|u| { - readable.contains(&physical.physical_candidates[u.0].logical_candidate_id) - }) + .map(|&edge| physical.edge(edge).producer_candidate_id) + .filter(|&u| !reads_one_batch_of(nodes, v, u)) .filter_map(|u| usage[u.0].as_ref()?.as_ref().ok()), ) }) .collect() } -/// The input groups of candidate `v`, by input index. -fn input_groups( - memo: &Memo, - physical: &PhysicalASAPMemoCandidates, - v: PhysicalCandidateId, -) -> Vec { - memo.candidate(physical.physical_candidates[v.0].logical_candidate_id) - .expression - .operator - .children() - .into_iter() - .copied() - .collect() +/// Whether consumer `c` reads only one batch of producer `u`'s output: `c` +/// runs at ingestion time and `u` is a raw `Scan` materialized there, whose +/// output is the rows it retains, what a query-time consumer reads. Such an +/// edge costs what reading a `Scan` that is not materialized does. +fn reads_one_batch_of( + nodes: &[PhysicalASAPDAGNode], + c: PhysicalCandidateId, + u: PhysicalCandidateId, +) -> bool { + let ingestion_time = + |v: PhysicalCandidateId| nodes[v.0].output_state.timing == ExecutionTiming::IngestionTime; + ingestion_time(c) + && ingestion_time(u) + && nodes[u.0].retains_rows() + && matches!( + nodes[u.0].payload, + Operator::NonASAP(NonASAPOp::Scan { .. }) + ) } /// An input's logical output over its readable costed candidates' `usages`: the most @@ -280,69 +549,73 @@ fn logical_output<'a: 'b, 'b>( output } -/// The usage of the candidate `node` over `inputs` (module doc): run -/// [`executions_per_second`] for the queries reaching its group, over the -/// window an ingestion-time grouping maintains +/// Candidate `v`'s usage as `node` over `inputs`, computed `per_second` +/// times per second, over the window an ingestion-time grouping maintains /// ([`asap_cost::maintained_window_ms`], over the memo's groups) or, for a /// query-time `Scan`, `scan_read_ms` per execution. +#[allow(clippy::too_many_arguments)] fn cost_node<'a>( physical: &PhysicalASAPMemoCandidates, + v: PhysicalCandidateId, node: &'a PhysicalASAPDAGNode, inputs: &[ProducerOutput<'a>], - entries: &[QueryWorkloadEntry], + per_second: f64, + scan_read_ms: Option, data: &DataWorkload, cost_inputs: &'a CostInputs, - scan_read_ms: Option, ) -> Result, String> { let memo = &physical.logical_candidates.memo; - let candidate = node - .selected_physical_candidate - .expect("a candidate's node names it"); - let group = physical.physical_candidates[candidate.0] - .logical_candidate_id - .group; - let per_second = executions_per_second( - node.output_state.timing, - node.lifetime, - memo.groups()[group.0 as usize] - .reached_by_queries() - .iter() - .filter_map(|&query| entries.get(query)), - cost_inputs.non_repeating_query_horizon_seconds, - ) - .map_err(|error| error.to_string())?; + let group = physical.physical_candidates[v.0].logical_candidate_id.group; let window_ms = match node.output_state.timing { ExecutionTiming::QueryTime => scan_read_ms, - // A chain's group computes the chain (Pass 2 adds chain candidates - // only to it), so the group's window is the chain's. - ExecutionTiming::IngestionTime => maintained_window_ms( - group, - |group: MemoGroupId| &memo.groups()[group.0 as usize].expression, - |group| { - memo.groups()[group.0 as usize] - .expression - .children() - .first() - .map(|g| **g) - }, - ), + // A chain maintains its own window; any other grouping the first + // range below its group. + ExecutionTiming::IngestionTime => physical.physical_candidates[v.0] + .chain_tumbling_window_width_ms(physical) + .or_else(|| { + maintained_window_ms( + group, + |group: MemoGroupId| &memo.groups()[group.0 as usize].expression, + |group| { + memo.groups()[group.0 as usize] + .expression + .children() + .first() + .map(|g| **g) + }, + ) + }), }; node_resource_usage_and_cost(node, inputs, (1, per_second), window_ms, data, cost_inputs) .map_err(|error| error.to_string()) } -/// $w_v$ from a candidate's usage: its cost per second under cost model 1 -/// (the raw data a query-time scan keeps on disk is charged per source, -/// [`SelectionCosts::raw_data_on_disk`]). -fn cost_per_second(usage: &NodeResourceUsageAndCost<'_>) -> Result { - let cost = usage +/// A candidate's cost per second from its usage under cost model 1: its +/// own, and the raw data a query-time scan keeps on disk +/// ([`asap_cost::raw_data_disk_cost_per_second`]). +fn cost_per_second( + usage: &NodeResourceUsageAndCost<'_>, + cost_inputs: &CostInputs, +) -> Result<(f64, f64), String> { + let own = usage .cost .cost_per_second .ok_or("the selection MILP costs under cost model 1 only")?; - match cost.is_finite() && cost >= 0.0 { - true => Ok(cost), + let raw_data = match usage.raw_scan { + Some((_, bytes)) => raw_data_disk_cost_per_second(bytes, cost_inputs) + .ok_or("the selection MILP costs under cost model 1 only")?, + None => 0.0, + }; + match [own, raw_data] + .iter() + .all(|cost| cost.is_finite() && *cost >= 0.0) + { + true => Ok((own, raw_data)), false => Err(format!( - "its cost per second {cost} is not a finite non-negative number" + "its cost per second {own} + {raw_data} is not a finite non-negative number" )), } } + +#[cfg(test)] +mod tests; diff --git a/crates/stage3-plan-selection/src/cost/tests.rs b/crates/stage3-plan-selection/src/cost/tests.rs new file mode 100644 index 000000000..e6c8a84dd --- /dev/null +++ b/crates/stage3-plan-selection/src/cost/tests.rs @@ -0,0 +1,224 @@ +//! Tests of [`super::execution_costs`]. + +use std::time::Duration; + +use asap_types::ir::operator::Reduction; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, TimeRangeKind}; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalEdge, + PhysicalProperties, StorageMedium, +}; +use asap_types::summary_type::{parameter_list, SketchAlgorithm, SketchParams, SummaryUpdate}; +use asap_types::test_support::memo_builders::{group, query, scan_group, try_candidate}; +use asap_types::workload::{ + Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, + TimestampMs, +}; + +use super::*; +use crate::retention::edge_retention_need_ms; +use crate::selectable::ExecutionContexts; + +const DAY_MS: u64 = 86_400_000; + +fn range(days: u64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(days * DAY_MS), + kind: TimeRangeKind::Range, + child, + }) +} + +/// The group of `operator`, with its own candidate at θ `parameters`. +fn kll_group( + memo: &mut Memo, + operator: Operator, + parameters: &SketchParams, +) -> MemoGroupId { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + try_candidate(memo, group, operator, Some(parameters.clone())).unwrap(); + group +} + +/// Example 3a's shape: a KLL state over the last five years of +/// 1,000,000 series sampled every 15 s, once a day, either built over +/// the scan's five years or merged from the windows of a 1-day chain +/// built at ingestion time; a second daily query reads the chain's +/// newest day of windows. The scan has a query-time candidate, one at +/// ingestion time and one materialized there (C0, C1, C2); the chain +/// (C3) may read C1 or C2, the five-year range (C4) C0 or C2. +/// +/// An ingestion-time consumer reads one batch of the scan, from C1 or +/// C2 alike: the chain's baseline input is that batch, not the five +/// years C2 retains for its query-time consumer. Selecting an +/// execution, one of its edges and the least retention its consumers +/// may need then never costs less than nothing (was −1,356,902/s: the +/// chain costed over five years of rows, the edge from C1 −22,398,978 +/// and a retention cost per ms inflated by those rows). +#[test] +fn no_execution_reading_a_producer_costs_less_than_nothing() { + let kll = ¶meter_list(SketchAlgorithm::Kll)[6]; + let summary_type = FieldDataType::Sketch(SketchAlgorithm::Kll); + let mut memo = Memo::new(&DataWorkload::default()); + let scan = scan_group(&mut memo, "m", &[], vec![]); + let chain = kll_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: summary_type.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: DAY_MS, + }), + kll, + ); + let five_years = 5 * 365; + // A `TimeRange` over the chain, and the merge over it, carry its θ. + let windows = kll_group(&mut memo, range(five_years, chain), kll); + let newest_day = kll_group(&mut memo, range(1, chain), kll); + let last_five_years = group(&mut memo, range(five_years, scan)); + let state = kll_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: last_five_years, + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }), + kll, + ); + try_candidate( + &mut memo, + state, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![windows], + group_by: None, + }), + Some(kll.clone()), + ) + .unwrap(); + for root in [state, newest_day] { + query(&mut memo, root, None); + } + use ExecutionTiming::{IngestionTime, QueryTime}; + let materialized = Lifetime::AcrossEvaluations(StorageMedium::Memory); + let candidate = |group, index_in_group, timing, lifetime| PhysicalCandidate { + logical_candidate_id: CandidateForMemoGroupId { + group, + index_in_group, + }, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + }; + let candidates = vec![ + candidate(scan, 0, QueryTime, Lifetime::None), + candidate(scan, 0, IngestionTime, Lifetime::None), + candidate(scan, 0, IngestionTime, materialized), + candidate(chain, 0, IngestionTime, materialized), + candidate(last_five_years, 0, QueryTime, Lifetime::None), + candidate(windows, 0, QueryTime, Lifetime::None), + candidate(state, 0, QueryTime, Lifetime::None), + candidate(state, 1, QueryTime, Lifetime::None), + candidate(newest_day, 0, QueryTime, Lifetime::None), + ]; + let edge = |u: usize, c: usize, admissible: bool| PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(u), + consumer_candidate_id: PhysicalCandidateId(c), + input_index_of_consumer: 0, + satisfies_logical_requirement: true, + satisfies_physical_requirement: admissible, + }; + let edges = vec![ + edge(0, 3, false), + edge(1, 3, true), + edge(2, 3, true), + edge(0, 4, true), + edge(1, 4, false), + edge(2, 4, true), + edge(3, 5, true), + edge(3, 8, true), + edge(4, 6, true), + edge(5, 7, true), + ]; + let groups = memo.groups().len(); + let physical = PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group: vec![None; groups], + exact_only_groups_of_query: vec![], + }, + candidates, + edges, + vec![], + ); + let daily = QueryWorkloadEntry { + query: Query("daily".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::default(), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(DAY_MS as u32), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, + }; + let entries = vec![daily.clone(), daily]; + let data: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival": {"ingesting": {"rows_per_second": 1_000_000.0 / 15.0}}, + "sources": {"m": {"series_count": 1_000_000, "sample_interval": 15_000}} + })) + .unwrap(); + let (contexts, query_outputs) = ExecutionContexts::new( + &physical, + &entries, + &[(QueryId(0), state), (QueryId(1), newest_day)], + ); + let costs = execution_costs( + &physical, + &contexts, + &query_outputs, + &entries, + &data, + &CostInputs::default(), + 0.0, + ); + // π_v times the least any consumer of `v` reads back, the least + // retention (R) lets a plan selecting `v` keep. + let least_retention = |v: PhysicalCandidateId| { + let least_need_ms = contexts + .edges() + .iter() + .filter(|(_, _, producer)| producer.candidate == v) + .filter_map(|&(_, edge, _)| edge_retention_need_ms(&physical, &entries, edge)) + .min() + .unwrap_or(0); + costs + .retention_cost_per_second + .get(&v) + .copied() + .unwrap_or(0.0) + * least_need_ms as f64 + }; + let mut checked = 0; + for (index, &(consumer, edge, _)) in contexts.edges().iter().enumerate() { + let (Ok(execution), Some(edge_cost)) = ( + &costs.per_execution[contexts.index_of(consumer)], + costs.per_edge[index], + ) else { + continue; + }; + let total = execution + edge_cost + least_retention(consumer.candidate); + assert!( + total >= 0.0, + "{:?} reading through {edge:?}: {execution} + {edge_cost} + retention = {total}", + consumer.candidate + ); + checked += 1; + } + // Among them the chain reading either scan. + assert!(checked >= 2, "{checked} edges costed"); +} diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index a7cc68613..4d44bc010 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -4,17 +4,20 @@ //! 4 and 5). //! //! [`select_plan`] reads Stage 2's physical candidates $\mathcal C$ and the -//! selectable part of the physical edges $\mathcal E$ ([`selectable`]), -//! costs them with `asap-cost` ([`cost`]), selects with a MILP over $x_v$, -//! $y_{u,c,i}$, $z_{q,v}$, $t_v$ and $r_s$ ([`milp`]; constraints (A)–(F) -//! and the one-consumer rule; (G) and (H) hold because $y$ exists only on -//! admissible edges), and builds the plan (`PhysicalASAPDAG`) from -//! $\{x = 1\}$ and $\{y = 1\}$ as its last step ([`plan`], with each -//! node's retention from [`retention`]). The extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, -//! $x_{\kappa,v}$, $y_{\kappa,u,c,i}$ and the retention amount $h_v$ -//! (section 5), and accuracy at each query root, constraint (K), with -//! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from -//! `asap-cost`. +//! admissible part of the physical edges $\mathcal E$, finds every +//! execution of a candidate in an execution context $\kappa$ and every edge +//! between executions the plan may select ([`selectable`]), costs them with +//! `asap-cost` ([`cost`]), and selects with a MILP ([`milp`]) over +//! $x_{\kappa,v}$ per execution (an `AcrossEvaluations` candidate runs +//! once, $x^{\mathrm{mat}}_v$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ and +//! the retention $h_v$, under constraints (A)–(F) per context, the +//! one-consumer rule and (R) ((G) and (H) hold because $y$ exists only on +//! admissible edges whose producer delivers on the consumer's context). It +//! builds the plan (`PhysicalASAPDAG`) from the selected executions and +//! edges as its last step ([`plan`], each node retaining its candidate's +//! $h_v$, the needs from [`retention`]). Accuracy at each query root, +//! constraint (K), with per-candidate error bounds $E_c$ and $D_c$ +//! (section 4), is the remaining extension. Costs come from `asap-cost`. mod cost; mod milp; @@ -28,11 +31,11 @@ use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::stages::{PhysicalASAPMemoCandidates, PruneReason, PrunedCandidate}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::cost::selection_costs; -use crate::milp::model::{solve_selection, SelectionInputs, SelectionMilpError}; +use crate::cost::execution_costs; +use crate::milp::model::{solve_selection, QueryId, SelectionInputs, SelectionMilpError}; use crate::plan::plan_of; use crate::report::{accepted_accuracy, infeasible}; -use crate::selectable::{output_groups, query_outputs, selectable_edges}; +use crate::selectable::{output_groups, Execution, ExecutionContexts}; pub use report::{ large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, @@ -71,31 +74,43 @@ pub fn select_plan( entries: entries.len(), }); } - let selectable = selectable_edges(physical, entries); + let outputs = output_groups(physical); + let (contexts, output_executions) = ExecutionContexts::new(physical, entries, &outputs); let request_overhead_cost_per_second = request_overhead_cost_per_second(entries, cost_inputs) .map_err(|error| ErrorSelectingPlan::Cost { candidate: "every plan (the request overhead)".into(), node: 0, error, })?; - let costs = selection_costs( + let costs = execution_costs( physical, - &selectable, + &contexts, + &output_executions, entries, data, cost_inputs, request_overhead_cost_per_second, ); - let outputs = output_groups(physical); - let query_outputs = query_outputs(physical, entries, &costs, &outputs); + let query_outputs: Vec<(QueryId, Vec)> = output_executions + .into_iter() + .map(|(query, executions)| { + let costed = executions + .into_iter() + .filter(|&execution| costs.per_execution[contexts.index_of(execution)].is_ok()) + .collect(); + (query, costed) + }) + .collect(); if query_outputs .iter() - .any(|(_, candidates)| candidates.is_empty()) + .any(|(_, executions)| executions.is_empty()) { return Err(infeasible(&costs)); } let solved = solve_selection(&SelectionInputs { physical, + entries, + contexts: &contexts, costs: &costs, query_outputs: &query_outputs, }) @@ -103,16 +118,21 @@ pub fn select_plan( SelectionMilpError::Infeasible => infeasible(&costs), SelectionMilpError::Solver(reason) => ErrorSelectingPlan::Solver(reason), })?; - let plan = plan_of(physical, entries, &solved, &outputs).map_err(|error| { + let plan = plan_of(physical, &solved, &outputs).map_err(|error| { ErrorSelectingPlan::Solver(format!("the selected plan is invalid: {error}")) })?; + // A candidate none of whose executions could be costed. + let mut costed = vec![None; physical.physical_candidates.len()]; + for (execution, cost) in contexts.executions().iter().zip(&costs.per_execution) { + let known = costed[execution.candidate.0].get_or_insert(false); + *known |= cost.is_ok(); + } let selection = Selection { accepted_accuracy_per_query: accepted_accuracy(physical, &plan), - pruned_candidates: costs - .candidate_cost_per_second + pruned_candidates: costed .iter() .zip(&physical.physical_candidates) - .filter(|(cost, _)| cost.is_err()) + .filter(|(costed, _)| **costed == Some(false)) .map(|(_, &candidate)| PrunedCandidate { candidate, reason: PruneReason::UsageNotCostable, diff --git a/crates/stage3-plan-selection/src/milp/execution_contexts.rs b/crates/stage3-plan-selection/src/milp/execution_contexts.rs index e1b66418f..3e4d7b64a 100644 --- a/crates/stage3-plan-selection/src/milp/execution_contexts.rs +++ b/crates/stage3-plan-selection/src/milp/execution_contexts.rs @@ -1,40 +1,108 @@ //! The execution-context extension of the selection MILP //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 5): its variables and constraint (R), `todo!()` until execution -//! contexts are selected. - -use std::collections::HashMap; - -use good_lp::{Constraint, Variable}; - -use asap_types::stages::{ - ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, -}; - -/// The execution-context variables (section 5), which replace $x$ and the -/// unindexed $y$ of the baseline. -#[allow(dead_code)] // Read once execution contexts are selected. -struct ExecutionContextVariables { - /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. - x_materialized: HashMap, - /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. - x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, - /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. - y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, - /// $h_v$, for an `AcrossEvaluations` producer. - retention_amount: HashMap, +//! section 5): the retention amount $h_v$ and constraint (R). The +//! executions $x_{\kappa,v}$ and edges $y_{\kappa,u,c,i}$ it selects over +//! are [`crate::selectable::ExecutionContexts`]'s, in [`super::model`]. + +use std::collections::{BTreeMap, HashMap}; + +use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; + +use asap_types::stages::PhysicalCandidateId; + +use super::model::{QueryId, SelectionInputs}; +use crate::retention::{edge_retention_need_ms, output_retention_need_ms}; +use crate::selectable::Execution; + +/// $h_v$ in seconds (the unit keeps (R)'s coefficients near the others'), +/// for each candidate with a retention cost +/// ([`crate::cost::ExecutionCosts::retention_cost_per_second`]), added to +/// `problem`. +pub(crate) fn retention_variables( + inputs: &SelectionInputs<'_>, + problem: &mut ProblemVariables, +) -> BTreeMap { + let mut retaining: Vec = inputs + .costs + .retention_cost_per_second + .keys() + .copied() + .collect(); + retaining.sort_unstable(); + retaining + .into_iter() + .map(|v| (v, problem.add(variable().min(0)))) + .collect() } -/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$: for a -/// time-indexed producer (raw data, a chain, results stored per evaluation -/// time) the reading path's offset plus range, a constant per edge, so (R) -/// is linear. Query latency is ignored: a result counts as read when its -/// evaluation starts. -#[allow(dead_code)] // Called once execution contexts are selected. -fn retention_constraints( - physical: &PhysicalASAPMemoCandidates, - variables: &ExecutionContextVariables, +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$ and +/// $h_v \ge \operatorname{span}_q\, z_{q,v}$, for a candidate $v$ with $h_v$: +/// how far back its rows are read through each selectable edge `y` (the +/// reading path's offset plus range, [`edge_retention_need_ms`]) and by +/// each query reading it directly (its range-evaluation span), a constant +/// per edge, so (R) is linear. Query latency is ignored: a result counts as +/// read when its evaluation starts. +pub(crate) fn retention_constraints( + inputs: &SelectionInputs<'_>, + h: &BTreeMap, + y: &BTreeMap, + z: &BTreeMap<(QueryId, Execution), Variable>, ) -> Vec { - let _ = (physical, variables); - todo!("#1207 PR 7: constraint (R)") + let mut constraints = Vec::new(); + for (&position, &y) in y { + let (_, edge, producer) = inputs.contexts.edges()[position]; + let Some(&h) = h.get(&producer.candidate) else { + continue; + }; + let need_ms = edge_retention_need_ms(inputs.physical, inputs.entries, edge) + .expect("a candidate with h_v is read within a bound"); + constraints.push(constraint!(h >= seconds(need_ms) * y)); + } + for (&(query, execution), &z) in z { + if let Some(&h) = h.get(&execution.candidate) { + let need_ms = output_retention_need_ms(&inputs.entries[query.0]); + constraints.push(constraint!(h >= seconds(need_ms) * z)); + } + } + constraints +} + +/// The least $h_v$ (R) allows for a selection, in ms, of each candidate +/// with $h_v$ that `edges` (selected, by position in +/// `ExecutionContexts::edges`) or `outputs` (the query outputs and the +/// executions answering them) read: what the MILP's optimum retains when +/// $\pi_v > 0$, and what the plan retains. +pub(crate) fn least_retention_ms( + inputs: &SelectionInputs<'_>, + edges: impl IntoIterator, + outputs: impl IntoIterator, +) -> HashMap { + let retaining = &inputs.costs.retention_cost_per_second; + let mut retention_ms: HashMap = HashMap::new(); + let mut read = |v: PhysicalCandidateId, need_ms: u64| { + if retaining.contains_key(&v) { + let most = retention_ms.entry(v).or_default(); + *most = (*most).max(need_ms); + } + }; + for position in edges { + let (_, edge, producer) = inputs.contexts.edges()[position]; + if retaining.contains_key(&producer.candidate) { + let need_ms = edge_retention_need_ms(inputs.physical, inputs.entries, edge) + .expect("a candidate with h_v is read within a bound"); + read(producer.candidate, need_ms); + } + } + for (query, execution) in outputs { + read( + execution.candidate, + output_retention_need_ms(&inputs.entries[query.0]), + ); + } + retention_ms +} + +/// `ms` in seconds, the unit of $h_v$ in the MILP. +fn seconds(ms: u64) -> f64 { + ms as f64 / 1_000.0 } diff --git a/crates/stage3-plan-selection/src/milp/graph.rs b/crates/stage3-plan-selection/src/milp/graph.rs index ec2da8fd5..cc491999b 100644 --- a/crates/stage3-plan-selection/src/milp/graph.rs +++ b/crates/stage3-plan-selection/src/milp/graph.rs @@ -1,32 +1,34 @@ //! The strongly connected components of the selectable edges between -//! candidates: acyclicity (E) needs an order variable $t_v$ only inside a +//! executions: acyclicity (E) needs an order variable $t$ only inside a //! component, bounded by its size. use std::collections::HashMap; use super::model::SelectionInputs; -/// Each candidate's strongly connected component of the selectable edges -/// (those [`crate::cost::SelectionCosts::edge_cost_change_per_second`] costs), computed once per model. +/// Each execution's strongly connected component of the selectable edges +/// (those [`crate::cost::ExecutionCosts::per_edge`] costs), computed once +/// per model, by position in `ExecutionContexts::executions`. pub(crate) struct Components { - /// Each candidate's component number. + /// Each execution's component number. pub(crate) of: Vec, - /// The size of each candidate's component: 1 for a candidate on no + /// The size of each execution's component: 1 for an execution on no /// cycle. pub(crate) size: Vec, } impl Components { pub(crate) fn of(inputs: &SelectionInputs<'_>) -> Self { + let contexts = inputs.contexts; let of = strongly_connected_components( - inputs.physical.physical_candidates.len(), - inputs - .costs - .edge_cost_change_per_second - .keys() - .map(|&edge| { - let edge = inputs.physical.edge(edge); - (edge.producer_candidate_id.0, edge.consumer_candidate_id.0) + contexts.executions().len(), + contexts + .edges() + .iter() + .zip(&inputs.costs.per_edge) + .filter(|(_, cost)| cost.is_some()) + .map(|(&(consumer, _, producer), _)| { + (contexts.index_of(producer), contexts.index_of(consumer)) }), ); let mut count: HashMap = HashMap::new(); diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index 3b30ee22c..2e893922d 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -1,9 +1,10 @@ //! Stage 3's selection MILP //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! sections 3–5): the baseline's variables and one function per constraint -//! family ([`model`]), the graph algorithm it needs ([`graph`]), the bound -//! it scales by ([`upper_bound`]), and the execution-context and accuracy -//! extensions ([`execution_contexts`], [`accuracy`]). +//! sections 3–5): its variables over execution contexts and one function +//! per constraint family ([`model`]), the retention amount $h_v$ and +//! constraint (R) ([`execution_contexts`]), the graph algorithm it needs +//! ([`graph`]), the bound it scales by ([`upper_bound`]), and the accuracy +//! extension ([`accuracy`]). mod accuracy; mod execution_contexts; mod graph; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index ce2d5c7cf..188433c32 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -4,15 +4,15 @@ //! //! | Family | Variables | Constraints | //! |---|---|---| -//! | baseline (section 3) | $x_v$, $y_{u,c,i}$, $z_{q,v}$, $t_v$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_constraints`]; (G), (H): $y$ exists only on admissible edges | -//! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ (`ExecutionContextVariables`) | (R) `retention_constraints` | +//! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_constraints`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | +//! | retention (section 5) | $h_v$ | (R) [`super::execution_contexts::retention_constraints`] | //! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ (`AccuracyVariables`) | (K) `accuracy_constraints` | //! -//! The baseline is solved by [`solve_selection`]; the extensions' -//! variables and constraints are in [`super::execution_contexts`] and -//! [`super::accuracy`], `todo!()` until they are added. The strongly -//! connected components (E) needs are in [`super::graph`]; the bound that -//! leaves out candidates before scaling in [`super::upper_bound`]. +//! [`solve_selection`] solves it without (K), whose variables and +//! constraint are in [`super::accuracy`], `todo!()` until it is added. The +//! strongly connected components (E) needs are in [`super::graph`]; the +//! bound that leaves out executions before scaling in +//! [`super::upper_bound`]. use std::collections::{BTreeMap, HashMap}; use std::time::Instant; @@ -24,49 +24,61 @@ use good_lp::{ use asap_types::ir::operator::Source; use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::Lifetime; -use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; +use asap_types::stages::{ + Lifetime, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, +}; +use asap_types::workload::QueryWorkloadEntry; +use super::execution_contexts::{least_retention_ms, retention_constraints, retention_variables}; use super::graph::Components; use super::upper_bound::dearer_than_a_feasible_plan; -use crate::cost::SelectionCosts; +use crate::cost::ExecutionCosts; +use crate::selectable::{Execution, ExecutionContexts}; /// A query, by index in the workload. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueryId(pub usize); -/// What the baseline MILP is built over. +/// What the MILP is built over. pub(crate) struct SelectionInputs<'a> { pub(crate) physical: &'a PhysicalASAPMemoCandidates, - /// $w_v$ and $c_{u,c,i}$: $y$ exists only on the edges - /// [`SelectionCosts::edge_cost_change_per_second`] costs, $x_v$ is 0 for a - /// candidate it could not cost. - pub(crate) costs: &'a SelectionCosts, + /// Each query's workload entry, for the retention needs of (R). + pub(crate) entries: &'a [QueryWorkloadEntry], + /// The executions $x$ and edges $y$ are over. + pub(crate) contexts: &'a ExecutionContexts, + /// The objective's coefficients: $x$ is 0 for an execution it could + /// not cost, $y$ exists only on an edge it costs, $h_v$ only for a + /// candidate with a retention cost. + pub(crate) costs: &'a ExecutionCosts, /// Each query output reference (a query, or one operator a scalar query - /// reads), with the candidates that may answer it: $z_{q,v}$ exists + /// reads), with the executions that may answer it: $z_{q,v}$ exists /// only for them. - pub(crate) query_outputs: &'a [(QueryId, Vec)], + pub(crate) query_outputs: &'a [(QueryId, Vec)], } -/// The baseline variables (section 3). +/// The variables (sections 3 and 5). pub(crate) struct SelectionVariables { - /// $x_v$, by [`PhysicalCandidateId`]. + /// $x_{\kappa,v}$, by position in `ExecutionContexts::executions`. x: Vec, - /// $y_{u,c,i}$, on admissible edges only, in edge order. - y: BTreeMap, - /// $z_{q,v}$, for $v \in G_q$, in key order. - z: BTreeMap<(QueryId, PhysicalCandidateId), Variable>, - /// $t_v$, by [`PhysicalCandidateId`]: only for candidates on a cycle of - /// admissible edges (a strongly connected component); `None` elsewhere. + /// $y_{\kappa,u,c,i}$, by position in `ExecutionContexts::edges`, on + /// the edges costed between executions kept, in edge order. + y: BTreeMap, + /// $z_{q,v}$, by the execution answering $q$, in key order. + z: BTreeMap<(QueryId, Execution), Variable>, + /// $t$, by position in `ExecutionContexts::executions`: only for + /// executions on a cycle of selectable edges (a strongly connected + /// component); `None` elsewhere. t: Vec>, + /// $h_v$ in seconds, for a candidate with a retention cost. + h: BTreeMap, /// Per raw-data source a query-time scan keeps on disk at a cost: the /// cost per second of keeping it, at least each selected scan's /// ([`raw_data_disk_constraints`]). raw_data_on_disk: BTreeMap, } -/// $x$, $y$, $z$, $t$ and $r$ over `inputs`, added to `problem`: $x_v$ -/// fixed at 0 for a candidate that could not be costed or is `left_out`, +/// $x$, $y$, $z$, $t$, $h$ and $r$ over `inputs`, added to `problem`: $x$ +/// fixed at 0 for an execution that could not be costed or is `left_out`, /// and no $y$ on an edge of one. fn selection_variables( inputs: &SelectionInputs<'_>, @@ -74,46 +86,46 @@ fn selection_variables( components: &Components, problem: &mut ProblemVariables, ) -> SelectionVariables { - let count = inputs.physical.physical_candidates.len(); - let x = (0..count) - .map( - |v| match inputs.costs.candidate_cost_per_second[v].is_ok() && !left_out[v] { - true => problem.add(variable().binary()), - false => problem.add(variable().min(0).max(0)), - }, - ) - .collect(); - let mut edges: Vec = inputs + let contexts = inputs.contexts; + let x = inputs .costs - .edge_cost_change_per_second - .keys() - .copied() - .filter(|&edge| { - let edge = inputs.physical.edge(edge); - !left_out[edge.producer_candidate_id.0] && !left_out[edge.consumer_candidate_id.0] + .per_execution + .iter() + .zip(left_out) + .map(|(cost, &left_out)| match cost.is_ok() && !left_out { + true => problem.add(variable().binary()), + false => problem.add(variable().min(0).max(0)), }) .collect(); - edges.sort_unstable(); - let y = edges - .into_iter() - .map(|edge| (edge, problem.add(variable().binary()))) + let y = contexts + .edges() + .iter() + .zip(&inputs.costs.per_edge) + .enumerate() + .filter(|(_, (&(consumer, _, producer), cost))| { + cost.is_some() + && !left_out[contexts.index_of(consumer)] + && !left_out[contexts.index_of(producer)] + }) + .map(|(position, _)| (position, problem.add(variable().binary()))) .collect(); let mut z = BTreeMap::new(); - for (query, candidates) in inputs.query_outputs { - for &v in candidates { - z.entry((*query, v)) + for (query, executions) in inputs.query_outputs { + for &execution in executions { + z.entry((*query, execution)) .or_insert_with(|| problem.add(variable().binary())); } } - // t_v only inside a strongly connected component of the selectable + // t only inside a strongly connected component of the selectable // edges, bounded by its size. let t = components .size .iter() .map(|&size| (size > 1).then(|| problem.add(variable().min(0).max((size - 1) as f64)))) .collect(); + let h = retention_variables(inputs, problem); let mut raw_data_on_disk = BTreeMap::new(); - for (source, _) in inputs.costs.raw_data_on_disk.iter().flatten() { + for source in inputs.costs.raw_data_source.values() { raw_data_on_disk .entry(source.clone()) .or_insert_with(|| problem.add(variable().min(0))); @@ -123,114 +135,92 @@ fn selection_variables( y, z, t, + h, raw_data_on_disk, } } -/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$. +/// (A) $\sum_{u \in P_{c,i}} y_{\kappa,u,c,i} = x_{\kappa,c}$, per execution +/// of $c$ and input $i$. fn input_satisfaction_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { let physical = inputs.physical; + let contexts = inputs.contexts; + let mut producers: HashMap<(usize, usize), Expression> = HashMap::new(); + for (&position, &y) in &variables.y { + let (consumer, edge, _) = contexts.edges()[position]; + *producers + .entry(( + contexts.index_of(consumer), + physical.edge(edge).input_index_of_consumer, + )) + .or_default() += y; + } let mut constraints = Vec::new(); - for c in (0..physical.physical_candidates.len()).map(PhysicalCandidateId) { - for i in 0..physical.input_count(c) { - let producers: Expression = physical - .admissible_edges_into(c, i) - .iter() - .filter_map(|edge| variables.y.get(edge)) - .sum(); - constraints.push(constraint!(producers == variables.x[c.0])); + for (index, execution) in contexts.executions().iter().enumerate() { + for i in 0..physical.input_count(execution.candidate) { + let chosen = producers.remove(&(index, i)).unwrap_or_default(); + constraints.push(constraint!(chosen == variables.x[index])); } } constraints } -/// (B) $y_{u,c,i} \le x_u$, except for an unmaterialized query-time -/// producer ([`is_unmaterialized_query_time`]), whose (F) row implies it. +/// (B) $y_{\kappa,u,c,i} \le x$ of $u$'s execution it reads. fn producer_selection_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { + let contexts = inputs.contexts; variables .y .iter() - .filter_map(|(&edge, &y)| { - let producer = inputs.physical.edge(edge).producer_candidate_id; - (!is_unmaterialized_query_time(inputs, producer.0)) - .then(|| constraint!(y <= variables.x[producer.0])) + .map(|(&position, &y)| { + let (_, _, producer) = contexts.edges()[position]; + constraint!(y <= variables.x[contexts.index_of(producer)]) }) .collect() } -/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x_v$, except for -/// an unmaterialized query-time candidate -/// ([`is_unmaterialized_query_time`]), whose (F) row implies it. +/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x$ of $v$'s +/// execution answering $q$. fn query_coverage_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { let mut constraints = Vec::new(); - for (query, candidates) in inputs.query_outputs { - let covered: Expression = candidates.iter().map(|&v| variables.z[&(*query, v)]).sum(); + for (query, executions) in inputs.query_outputs { + let covered: Expression = executions + .iter() + .map(|&execution| variables.z[&(*query, execution)]) + .sum(); constraints.push(constraint!(covered == 1)); } - for (&(_, v), &z) in &variables.z { - if !is_unmaterialized_query_time(inputs, v.0) { - constraints.push(constraint!(z <= variables.x[v.0])); - } + for (&(_, execution), &z) in &variables.z { + constraints.push(constraint!( + z <= variables.x[inputs.contexts.index_of(execution)] + )); } constraints } -/// Whether candidate `v` is a query-time result that is not materialized -/// ($\ell$ = None), so it has at most one consumer and its (F) row is the -/// equality $\sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v} = x_v$ -/// ([`use_constraints`]). With $y, z \ge 0$ that equality implies (B) -/// $y_{v,c,i} \le x_v$ and (D) $z_{q,v} \le x_v$, so those rows are left -/// out for it. -pub(super) fn is_unmaterialized_query_time(inputs: &SelectionInputs<'_>, v: usize) -> bool { - let properties = inputs.physical.physical_candidates[v].physical_properties; - properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime -} - -/// The cost per second of a plan of `candidates` reading over edges that -/// cost `edge_costs` ($c_{u,c,i}$), without the request overhead every plan -/// pays: the candidates' $w_v$, the edges' costs, and each source's raw -/// data on disk once, at its largest selected scan. -pub(super) fn plan_cost_per_second( - inputs: &SelectionInputs<'_>, - candidates: impl IntoIterator, - edge_costs: impl IntoIterator, -) -> f64 { - let mut raw_data_on_disk: BTreeMap<&Source, f64> = BTreeMap::new(); - let mut cost = 0.0; - for v in candidates { - if let Ok(own) = &inputs.costs.candidate_cost_per_second[v] { - cost += own; - } - if let Some((source, on_disk)) = &inputs.costs.raw_data_on_disk[v] { - let kept = raw_data_on_disk.entry(source).or_default(); - *kept = kept.max(*on_disk); - } - } - cost + edge_costs.into_iter().sum::() + raw_data_on_disk.values().sum::() -} - -/// (E) $t_u + 1 \le t_c + n(1 - y_{u,c,i})$, only on edges inside a strongly -/// connected component of the admissible edges, $n$ its size. +/// (E) $t_u + 1 \le t_c + n(1 - y_{\kappa,u,c,i})$ between the producer's +/// and the consumer's executions, only on edges inside a strongly connected +/// component of the selectable edges, $n$ its size. fn acyclicity_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, components: &Components, ) -> Vec { + let contexts = inputs.contexts; variables .y .iter() - .filter_map(|(&edge, &y)| { - let edge = inputs.physical.edge(edge); - let (u, c) = (edge.producer_candidate_id.0, edge.consumer_candidate_id.0); + .filter_map(|(&position, &y)| { + let (consumer, _, producer) = contexts.edges()[position]; + let (u, c) = (contexts.index_of(producer), contexts.index_of(consumer)); if components.of[u] != components.of[c] { return None; } @@ -241,13 +231,13 @@ fn acyclicity_constraints( .collect() } -/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$ for every -/// candidate; with it, a query-time result that is not materialized -/// ($\ell$ = None) has at most one consumer, $\sum_{(v,c,i)} y_{v,c,i} + -/// \sum_q z_{q,v} \le x_v$, so for such a candidate the two are one row, -/// $\sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v} = x_v$. Consumers that share a -/// result read a `OneEvaluation` or a materialized producer (section 5, per -/// execution context). An ingestion-time producer is exempt from the +/// (F) $x_{\kappa,v} \le \sum y + \sum_q z_{q,v}$ over the edges and query +/// outputs reading that execution; with it, a query-time result that is not +/// materialized ($\ell$ = None) has at most one consumer per context, +/// $\sum y + \sum_q z_{q,v} \le x_{\kappa,v}$, so for such an execution the +/// two are one row, $\sum y + \sum_q z_{q,v} = x_{\kappa,v}$. Consumers in +/// one context that share a result read a `OneEvaluation` or a materialized +/// producer (section 5). An ingestion-time producer is exempt from the /// one-consumer rule: there is no `OneEvaluation` at ingestion time, and the /// consumers of one ingestion run share its rows. fn use_constraints( @@ -257,67 +247,79 @@ fn use_constraints( uses_of(inputs, variables) .into_iter() .enumerate() - .map(|(v, used)| match is_unmaterialized_query_time(inputs, v) { - true => constraint!(used == variables.x[v]), - false => constraint!(variables.x[v] <= used), + .map(|(index, used)| { + let v = inputs.contexts.executions()[index].candidate; + let properties = inputs.physical.physical_candidates[v.0].physical_properties; + match properties.lifetime() == Lifetime::None + && properties.timing() == ExecutionTiming::QueryTime + { + true => constraint!(used == variables.x[index]), + false => constraint!(variables.x[index] <= used), + } }) .collect() } /// The raw data of each source is kept on disk once, for the selected scan -/// that keeps the most: $r_s \ge \mathrm{scale} \cdot d_v x_v$ for each -/// query-time `Scan` $v$ of source $s$, $d_v$ its cost per second of the -/// raw data it keeps. The objective's scale is in the row, so $r_s$ is the -/// scaled cost and its objective coefficient is 1. +/// that keeps the most: $r_s \ge \pi_v h_v$ for each query-time `Scan` $v$ +/// of source $s$ with $h_v$, and $r_s \ge d_v x_{\kappa,v}$ for each +/// execution of one without, $d_v$ its cost per second of the raw data it +/// keeps retaining everything. fn raw_data_disk_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, - objective_scale: f64, ) -> Vec { - inputs - .costs - .raw_data_on_disk - .iter() - .enumerate() - .filter_map(|(v, on_disk)| { - let (source, cost) = on_disk.as_ref()?; - Some(constraint!( - variables.raw_data_on_disk[source] >= *cost * objective_scale * variables.x[v] - )) - }) - .collect() + let costs = inputs.costs; + let mut scans: Vec<(&PhysicalCandidateId, &Source)> = costs.raw_data_source.iter().collect(); + scans.sort_unstable_by_key(|(v, _)| **v); + let mut constraints = Vec::new(); + for (v, source) in scans { + let r = variables.raw_data_on_disk[source]; + if let Some(&h) = variables.h.get(v) { + let per_second_retained = costs.retention_cost_per_second[v] * 1_000.0; + constraints.push(constraint!(r >= per_second_retained * h)); + } else if let Some(&d) = costs.unbounded_raw_data_on_disk_cost_per_second.get(v) { + for (index, execution) in inputs.contexts.executions().iter().enumerate() { + if execution.candidate == *v { + constraints.push(constraint!(r >= d * variables.x[index])); + } + } + } + } + constraints } -/// Per candidate, its selected edges to consumers and query outputs: -/// $\sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$. +/// Per execution, its selected edges to consumers and query outputs: +/// $\sum y + \sum_q z_{q,v}$. fn uses_of(inputs: &SelectionInputs<'_>, variables: &SelectionVariables) -> Vec { - let mut uses: Vec = (0..inputs.physical.physical_candidates.len()) - .map(|v| { - inputs - .physical - .admissible_edges_out_of(PhysicalCandidateId(v)) - .iter() - .filter_map(|edge| variables.y.get(edge)) - .sum() - }) - .collect(); - for (&(_, v), &z) in &variables.z { - uses[v.0] += z; + let contexts = inputs.contexts; + let mut uses: Vec = vec![Expression::default(); variables.x.len()]; + for (&position, &y) in &variables.y { + let (_, _, producer) = contexts.edges()[position]; + uses[contexts.index_of(producer)] += y; + } + for (&(_, execution), &z) in &variables.z { + uses[contexts.index_of(execution)] += z; } uses } -/// The baseline MILP's optimum: $\{x = 1\}$, $\{y = 1\}$ and $\{z = 1\}$, -/// and the MILP's size and time. +/// The MILP's optimum: the executions with $x = 1$, the edges with +/// $y = 1$, the execution answering each query output ($z = 1$) and the +/// retention of each selected candidate with $h_v$, and the MILP's size +/// and time. #[derive(Debug, Clone)] pub(crate) struct SolvedSelection { - /// The selected candidates, ascending. - pub(crate) candidates: Vec, - /// The selected edges, ascending. - pub(crate) edges: Vec, - /// The candidate answering each of `SelectionInputs::query_outputs`, in + /// The selected executions, ascending. + pub(crate) executions: Vec, + /// The selected edges as (consumer, edge, producer), ascending. + pub(crate) edges: Vec<(Execution, PhysicalEdgeId, Execution)>, + /// The execution answering each of `SelectionInputs::query_outputs`, in /// its order. - pub(crate) outputs: Vec, + pub(crate) outputs: Vec, + /// $h_v$ in ms of each selected candidate with $h_v$: the least (R) + /// allows for the selection. + pub(crate) retention_ms: HashMap, /// The optimum, unscaled: the selected plan's cost per second. pub(crate) objective_cost_per_second: f64, /// The MILP's variables and constraint rows. @@ -327,20 +329,22 @@ pub(crate) struct SolvedSelection { pub(crate) solve_ms: u64, } -/// Why the baseline MILP has no optimum. +/// Why the MILP has no optimum. #[derive(Debug, Clone, PartialEq)] pub(crate) enum SelectionMilpError { - /// No plan meets constraints (A)–(F). + /// No plan meets the constraints. Infeasible, /// The solver failed. Solver(String), } -/// Builds and solves the baseline MILP: minimize -/// $\sum_v w_v x_v + \sum_{(u,c,i)} c_{u,c,i} y_{u,c,i} + \sum_s r_s$ under -/// (A)–(F), the one-consumer rule ([`use_constraints`]) and -/// [`raw_data_disk_constraints`], with HiGHS at gap 0 on one thread. The -/// candidates dearer than a feasible plan are left out first +/// Builds and solves the MILP: minimize +/// $\sum_v W^{\mathrm{mat}}_v x^{\mathrm{mat}}_v + \sum_{\kappa,v} +/// \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum \rho_\kappa c_{\kappa,u,c,i} +/// y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s r_s$ ($\pi_v h_v$ of a +/// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], +/// [`raw_data_disk_constraints`] and (R), with HiGHS at gap 0 on one +/// thread. The executions dearer than a feasible plan are left out first /// ([`dearer_than_a_feasible_plan`]), then the coefficients are scaled so /// the largest left is 1,000 (HiGHS's tolerances are absolute). pub(crate) fn solve_selection( @@ -351,56 +355,68 @@ pub(crate) fn solve_selection( let components = Components::of(inputs); let mut problem = ProblemVariables::new(); let variables = selection_variables(inputs, &left_out, &components, &mut problem); - // Scaled by the largest coefficient of a candidate the bound keeps. - let kept = |v: &usize| !left_out[*v]; - let largest = (0..left_out.len()) - .filter(kept) - .filter_map(|v| inputs.costs.candidate_cost_per_second[v].as_ref().ok()) - .chain( - variables - .y - .keys() - .map(|edge| &inputs.costs.edge_cost_change_per_second[edge]), - ) - .chain( - (0..left_out.len()) - .filter(kept) - .filter_map(|v| inputs.costs.raw_data_on_disk[v].as_ref()) - .map(|(_, cost)| cost), - ) - .fold(0.0_f64, |largest, &cost| largest.max(cost.abs())); - let scale = match largest > 0.0 && largest.is_finite() { - true => 1_000.0 / largest, - false => 1.0, - }; let constraints: Vec = [ input_satisfaction_constraints(inputs, &variables), producer_selection_constraints(inputs, &variables), query_coverage_constraints(inputs, &variables), acyclicity_constraints(inputs, &variables, &components), use_constraints(inputs, &variables), - raw_data_disk_constraints(inputs, &variables, scale), + raw_data_disk_constraints(inputs, &variables), + retention_constraints(inputs, &variables.h, &variables.y, &variables.z), ] .into_iter() .flatten() .collect(); - let candidate_terms = (0..left_out.len()).filter(kept).filter_map(|v| { - let cost = inputs.costs.candidate_cost_per_second[v].as_ref().ok()?; - Some(cost * scale * variables.x[v]) - }); - let edge_terms = variables - .y + let costs = inputs.costs; + // π_v per second of retention, the unit of h. + let retention_cost = |v: &PhysicalCandidateId| costs.retention_cost_per_second[v] * 1_000.0; + // The objective's terms per second, unscaled: (cost, variable). A + // query-time scan's retention is in its source's r_s. + let terms: Vec<(f64, Variable)> = costs + .per_execution .iter() - .map(|(edge, &y)| inputs.costs.edge_cost_change_per_second[edge] * scale * y); - // Already scaled by its rows ([`raw_data_disk_constraints`]). - let raw_data_terms = variables - .raw_data_on_disk - .values() - .map(|&on_disk| Expression::from(on_disk)); - let objective: Expression = candidate_terms - .chain(edge_terms) - .chain(raw_data_terms) - .sum(); + .zip(&variables.x) + .zip(&left_out) + .filter(|(_, &left_out)| !left_out) + .filter_map(|((cost, &x), _)| Some((*cost.as_ref().ok()?, x))) + .chain( + variables + .y + .iter() + .map(|(&position, &y)| (costs.per_edge[position].expect("a costed edge"), y)), + ) + .chain( + variables + .h + .iter() + .filter(|(v, _)| !costs.raw_data_source.contains_key(v)) + .map(|(v, &h)| (retention_cost(v), h)), + ) + .chain(variables.raw_data_on_disk.values().map(|&r| (1.0, r))) + .collect(); + // Scaled by the largest coefficient left, the retention and raw-data + // bounds included. + let largest = terms + .iter() + .map(|(cost, _)| *cost) + .chain(costs.raw_data_source.keys().filter_map(|v| { + costs + .retention_cost_per_second + .get(v) + .map(|_| retention_cost(v)) + })) + .chain( + costs + .unbounded_raw_data_on_disk_cost_per_second + .values() + .copied(), + ) + .fold(0.0_f64, |largest, cost| largest.max(cost.abs())); + let scale = match largest > 0.0 && largest.is_finite() { + true => 1_000.0 / largest, + false => 1.0, + }; + let objective: Expression = terms.iter().map(|&(cost, v)| cost * scale * v).sum(); let variable_count = problem.len(); // HiGHS's columns are the variables in the order they were added. let column_of: HashMap = problem @@ -441,39 +457,48 @@ pub(crate) fn solve_selection( other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), }; let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; - // The optimum, unscaled, with the request overhead every plan pays. - let objective_cost_per_second = plan_cost_per_second( - inputs, - (0..variables.x.len()).filter(|&v| chosen(variables.x[v])), - variables - .y - .iter() - .filter(|&(_, &y)| chosen(y)) - .map(|(edge, _)| inputs.costs.edge_cost_change_per_second[edge]), - ) + inputs.costs.request_overhead_cost_per_second; + let contexts = inputs.contexts; + let selected: Vec = (0..variables.x.len()) + .filter(|&index| chosen(variables.x[index])) + .collect(); + let selected_edges: Vec = variables + .y + .iter() + .filter(|&(_, &y)| chosen(y)) + .map(|(&position, _)| position) + .collect(); + let outputs: Vec = inputs + .query_outputs + .iter() + .map(|(query, executions)| { + executions + .iter() + .copied() + .find(|&execution| chosen(variables.z[&(*query, execution)])) + .expect("(C): one output per query output reference") + }) + .collect(); + let answering: Vec<(QueryId, Execution)> = inputs + .query_outputs + .iter() + .zip(&outputs) + .map(|((query, _), &execution)| (*query, execution)) + .collect(); + let (cost_per_second, retention_ms) = + selection_cost_per_second(inputs, &selected, &selected_edges, &answering); + let objective_cost_per_second = cost_per_second + costs.request_overhead_cost_per_second; Ok(SolvedSelection { - objective_cost_per_second, - candidates: (0..variables.x.len()) - .filter(|&v| chosen(variables.x[v])) - .map(PhysicalCandidateId) - .collect(), - edges: variables - .y + executions: selected .iter() - .filter(|&(_, &y)| chosen(y)) - .map(|(&edge, _)| edge) + .map(|&index| contexts.executions()[index]) .collect(), - outputs: inputs - .query_outputs + edges: selected_edges .iter() - .map(|(query, candidates)| { - candidates - .iter() - .copied() - .find(|&v| chosen(variables.z[&(*query, v)])) - .expect("(C): one output per query output reference") - }) + .map(|&position| contexts.edges()[position]) .collect(), + outputs, + retention_ms, + objective_cost_per_second, variable_count, constraint_count, construction_ms, @@ -481,5 +506,54 @@ pub(crate) fn solve_selection( }) } +/// The cost per second of a selection, without the request overhead every +/// plan pays: its executions (`selected`, by position in +/// `ExecutionContexts::executions`) and edges (`edges`, by position in +/// `ExecutionContexts::edges`), the retention they and the query outputs +/// `answering` need ([`least_retention_ms`]), and each source's raw data +/// on disk at its largest selected scan; with that retention. +pub(crate) fn selection_cost_per_second( + inputs: &SelectionInputs<'_>, + selected: &[usize], + edges: &[usize], + answering: &[(QueryId, Execution)], +) -> (f64, HashMap) { + let costs = inputs.costs; + let retention_ms = least_retention_ms(inputs, edges.iter().copied(), answering.iter().copied()); + let mut raw_data_on_disk: BTreeMap<&Source, f64> = BTreeMap::new(); + let mut retention_cost_per_second = 0.0; + for (&v, &ms) in &retention_ms { + let cost = costs.retention_cost_per_second[&v] * ms as f64; + match costs.raw_data_source.get(&v) { + Some(source) => { + let kept = raw_data_on_disk.entry(source).or_default(); + *kept = kept.max(cost); + } + None => retention_cost_per_second += cost, + } + } + for &index in selected { + let v = inputs.contexts.executions()[index].candidate; + if let (Some(source), Some(&cost)) = ( + costs.raw_data_source.get(&v), + costs.unbounded_raw_data_on_disk_cost_per_second.get(&v), + ) { + let kept = raw_data_on_disk.entry(source).or_default(); + *kept = kept.max(cost); + } + } + let cost = selected + .iter() + .filter_map(|&index| costs.per_execution[index].as_ref().ok()) + .sum::() + + edges + .iter() + .filter_map(|&position| costs.per_edge[position]) + .sum::() + + retention_cost_per_second + + raw_data_on_disk.values().sum::(); + (cost, retention_ms) +} + #[cfg(test)] mod tests; diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 56c0bb7c7..9f5987bff 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -1,29 +1,39 @@ -//! The baseline MILP over small hand-built candidate graphs with given -//! costs: which candidates and edges its optimum selects, and the plan -//! built from it. +//! The selection MILP over small hand-built candidate graphs with given +//! costs: which executions and edges its optimum selects, its retention, +//! and the plan built from it. use std::collections::HashMap; +use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{Operator, QueryRoot}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::FieldDataType; +use asap_types::ir::{ASAPOp, Operator, QueryRoot}; use asap_types::stages::{ CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, }; use asap_types::stages::{Lifetime, PhysicalProperties, StorageMedium}; +use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; use asap_types::test_support::memo_builders::{ group, own_candidate, query, scan_group, try_candidate, }; use asap_types::test_support::memo_fixtures; use asap_types::workload::DataWorkload; +use asap_types::workload::{ + Predictability, Query, QueryRecurrence, QueryRequirements, RepeatedDemand, RepetitionInterval, + TimestampMs, +}; use super::*; -use crate::cost::SelectionCosts; +use crate::cost::ExecutionCosts; use crate::plan::plan_of; +use crate::selectable::ExecutionContexts; const NONE: Lifetime = Lifetime::None; const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); +const ACROSS_EVALUATIONS: Lifetime = Lifetime::AcrossEvaluations(StorageMedium::Memory); fn scan(memo: &mut Memo) -> MemoGroupId { scan_group(memo, "m", &["job"], vec![]) @@ -104,130 +114,176 @@ fn edge(physical: &PhysicalASAPMemoCandidates, u: usize, c: usize) -> PhysicalEd .0 } -/// Costs `w` per candidate and `edge_costs` per (producer, consumer), 0 for -/// every other edge. -fn costs( - physical: &PhysicalASAPMemoCandidates, - w: &[f64], - edge_costs: &[((usize, usize), f64)], -) -> SelectionCosts { - let mut edge_cost: HashMap = - physical.edges().map(|(id, _)| (id, 0.0)).collect(); - for &((u, c), cost) in edge_costs { - edge_cost.insert(edge(physical, u, c), cost); - } - SelectionCosts { - candidate_cost_per_second: w.iter().map(|&w| Ok(w)).collect(), - edge_cost_change_per_second: edge_cost, - raw_data_on_disk: vec![None; w.len()], - request_overhead_cost_per_second: 0.0, +/// Every query on a 10 s grid, `count` of them. +fn every_10s(count: usize) -> Vec { + (0..count).map(|_| every(10_000)).collect() +} + +/// A query every `interval_ms` at phase 0. +fn every(interval_ms: u32) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Query("q".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::default(), + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, } } -/// Each query output reference of `physical`'s memo with every candidate of -/// its group. -fn outputs(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, Vec)> { +/// Each query output reference of `physical`'s memo: query `q` reads its +/// operator root. +fn outputs(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, MemoGroupId)> { physical .logical_candidates .memo .queries() .iter() .enumerate() - .map(|(query, memo_query)| { - let QueryRoot::Operator(group) = memo_query.root else { - unreachable!("operator roots only") - }; - let candidates = (0..physical.physical_candidates.len()) - .filter(|&v| physical.physical_candidates[v].logical_candidate_id.group == group) - .map(PhysicalCandidateId) - .collect(); - (QueryId(query), candidates) + .map(|(query, memo_query)| match memo_query.root { + QueryRoot::Operator(group) => (QueryId(query), group), + QueryRoot::Scalar(_) => unreachable!("operator roots only"), }) .collect() } +/// Solves over `physical` for `entries` with every execution of candidate +/// `v` costing `w[v]`, every edge from `u` to `c` `edge_costs[(u, c)]` (0 +/// when missing) and candidate `v` retaining at `retention[v]` per ms; +/// returns the solution and the contexts it is over. fn solve( physical: &PhysicalASAPMemoCandidates, - costs: &SelectionCosts, -) -> Result { - solve_selection(&SelectionInputs { + entries: &[QueryWorkloadEntry], + w: &[f64], + edge_costs: &[((usize, usize), f64)], + retention: &[(usize, f64)], +) -> ( + Result, + ExecutionContexts, +) { + let (contexts, query_outputs) = ExecutionContexts::new(physical, entries, &outputs(physical)); + let costs = ExecutionCosts { + per_execution: contexts + .executions() + .iter() + .map(|execution| Ok(w[execution.candidate.0])) + .collect(), + per_edge: contexts + .edges() + .iter() + .map(|&(_, id, _)| { + let edge = physical.edge(id); + let key = (edge.producer_candidate_id.0, edge.consumer_candidate_id.0); + Some( + edge_costs + .iter() + .find(|(pair, _)| *pair == key) + .map_or(0.0, |(_, cost)| *cost), + ) + }) + .collect(), + retention_cost_per_second: retention + .iter() + .map(|&(v, per_ms)| (PhysicalCandidateId(v), per_ms)) + .collect(), + raw_data_source: HashMap::new(), + unbounded_raw_data_on_disk_cost_per_second: HashMap::new(), + request_overhead_cost_per_second: 0.0, + }; + let solved = solve_selection(&SelectionInputs { physical, - costs, - query_outputs: &outputs(physical), - }) + entries, + contexts: &contexts, + costs: &costs, + query_outputs: &query_outputs, + }); + (solved, contexts) } -fn ids(ids: &[usize]) -> Vec { - ids.iter().copied().map(PhysicalCandidateId).collect() +/// The candidates of `solved`'s executions, one per execution, ascending. +fn candidates(solved: &SolvedSelection) -> Vec { + let mut candidates: Vec = solved + .executions + .iter() + .map(|execution| execution.candidate.0) + .collect(); + candidates.sort_unstable(); + candidates } /// `m[5m]`, its scan with two physical candidates. Through v0 the read /// costs 1 + 5, through v1 2 + 0: the optimum takes v1, and the plan built /// from it is the scan v1 below the range v2, the scan retaining the 5 -/// minutes the range reads. +/// minutes the range reads (R). #[test] fn selects_the_producer_whose_own_and_edge_costs_are_lowest() { let mut memo = empty_memo(); let m = scan(&mut memo); let last_5m = range(&mut memo, 300, m); query(&mut memo, last_5m, None); - let first = own_candidate; let physical = physical( memo, &[ - (first(m), ONE_EVALUATION), - (first(m), NONE), - (first(last_5m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(m), NONE), + (own_candidate(last_5m), NONE), ], ); - let costs = costs(&physical, &[1.0, 2.0, 1.0], &[((0, 2), 5.0)]); - let solved = solve(&physical, &costs).unwrap(); - assert_eq!(solved.candidates, ids(&[1, 2])); - assert_eq!(solved.edges, vec![edge(&physical, 1, 2)]); - assert_eq!(solved.outputs, ids(&[2])); + let (solved, _) = solve( + &physical, + &every_10s(1), + &[1.0, 2.0, 1.0], + &[((0, 2), 5.0)], + &[(1, 1e-9)], + ); + let solved = solved.unwrap(); + assert_eq!(candidates(&solved), vec![1, 2]); + assert_eq!(solved.edges.len(), 1); + assert_eq!(solved.edges[0].1, edge(&physical, 1, 2)); + assert_eq!(solved.outputs[0].candidate, PhysicalCandidateId(2)); - let outputs = vec![(QueryId(0), last_5m)]; - let plan = plan_of(&physical, &[], &solved, &outputs).unwrap(); + let plan = plan_of(&physical, &solved, &outputs(&physical)).unwrap(); assert_eq!(plan.roots, vec![1]); assert_eq!(plan.edges.len(), 1); assert_eq!((plan.edges[0].producer, plan.edges[0].consumer), (0, 1)); - let candidates: Vec<_> = plan + let selected: Vec<_> = plan .nodes .iter() .map(|node| node.selected_physical_candidate) .collect(); assert_eq!( - candidates, + selected, vec![Some(PhysicalCandidateId(1)), Some(PhysicalCandidateId(2))] ); assert_eq!(plan.nodes[0].retention_ms, Some(300_000)); assert_eq!(plan.nodes[1].retention_ms, None); } -/// `m[5m]` and `m[1m]` both read the scan. Its candidate that is not -/// materialized (cost 1) may have one consumer, so the optimum reads the -/// scan's `OneEvaluation` candidate (cost 10) twice rather than both. +/// `m[5m]` and `m[1m]` both read the scan, both queries every 10 s: one +/// context. Its candidate that is not materialized (cost 1) may have one +/// consumer there, so the optimum reads the scan's `OneEvaluation` +/// candidate (cost 10) twice rather than both; without it no plan exists. #[test] -fn a_query_time_result_that_is_not_materialized_has_one_consumer() { +fn a_query_time_result_that_is_not_materialized_has_one_consumer_per_context() { let mut memo = empty_memo(); let m = scan(&mut memo); let last_5m = range(&mut memo, 300, m); let last_1m = range(&mut memo, 60, m); query(&mut memo, last_5m, None); query(&mut memo, last_1m, None); - let first = own_candidate; let physical = physical( memo, &[ - (first(m), NONE), - (first(last_5m), NONE), - (first(last_1m), NONE), - (first(m), ONE_EVALUATION), + (own_candidate(m), NONE), + (own_candidate(last_5m), NONE), + (own_candidate(last_1m), NONE), + (own_candidate(m), ONE_EVALUATION), ], ); - let with_one_evaluation = costs(&physical, &[1.0, 1.0, 1.0, 10.0], &[]); - let solved = solve(&physical, &with_one_evaluation).unwrap(); - assert_eq!(solved.candidates, ids(&[1, 2, 3])); + let (solved, _) = solve(&physical, &every_10s(2), &[1.0, 1.0, 1.0, 10.0], &[], &[]); + assert_eq!(candidates(&solved.unwrap()), vec![1, 2, 3]); let without_one_evaluation = PhysicalASAPMemoCandidates::new( physical.logical_candidates.clone(), @@ -239,11 +295,110 @@ fn a_query_time_result_that_is_not_materialized_has_one_consumer() { .collect(), vec![], ); - let only_unshared = costs(&without_one_evaluation, &[0.0, 1.0, 1.0], &[]); + let (solved, _) = solve( + &without_one_evaluation, + &every_10s(2), + &[0.0, 1.0, 1.0], + &[], + &[], + ); + assert_eq!(solved.unwrap_err(), SelectionMilpError::Infeasible); +} + +/// The same two queries every 10 s and every 15 s: two contexts, each +/// running the scan that is not materialized once for its own query +/// (cost 1 each), cheaper than one `OneEvaluation` run (cost 10), which +/// could not be shared across contexts anyway. +#[test] +fn a_result_that_is_not_materialized_runs_once_per_context_that_reads_it() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + let last_1m = range(&mut memo, 60, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_1m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(last_5m), NONE), + (own_candidate(last_1m), NONE), + (own_candidate(m), ONE_EVALUATION), + ], + ); + let (solved, contexts) = solve( + &physical, + &[every(10_000), every(15_000)], + &[1.0, 1.0, 1.0, 10.0], + &[], + &[], + ); + let solved = solved.unwrap(); + assert_eq!(candidates(&solved), vec![0, 0, 1, 2]); + let scan_contexts: Vec<_> = solved + .executions + .iter() + .filter(|execution| execution.candidate.0 == 0) + .map(|execution| execution.context) + .collect(); + assert_ne!(scan_contexts[0], scan_contexts[1]); + let plan = plan_of(&physical, &solved, &outputs(&physical)).unwrap(); + assert_eq!(plan.nodes.len(), 4); + assert_eq!(contexts.executions().len(), 6); +} + +/// `m[1m]` every 10 s and `m[5m]` every 20 s, both over a chain of 10 s +/// windows of the scan, materialized across evaluations: the chain runs +/// once, in its own context, and delivers on both grids; it retains the +/// longest range its selected consumers read, 5 minutes (R). +#[test] +fn a_materialized_chain_runs_once_and_retains_what_its_consumers_read_back() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let chain = group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: m, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 10_000, + }), + ); + let last_1m = range(&mut memo, 60, chain); + let last_5m = range(&mut memo, 300, chain); + query(&mut memo, last_1m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(chain), ACROSS_EVALUATIONS), + (own_candidate(last_1m), NONE), + (own_candidate(last_5m), NONE), + ], + ); + let (solved, _) = solve( + &physical, + &[every(10_000), every(20_000)], + &[1.0, 1.0, 1.0, 1.0], + &[], + &[(1, 1e-6)], + ); + let solved = solved.unwrap(); + assert_eq!(candidates(&solved), vec![0, 1, 2, 3]); assert_eq!( - solve(&without_one_evaluation, &only_unshared).unwrap_err(), - SelectionMilpError::Infeasible + solved.retention_ms.get(&PhysicalCandidateId(1)), + Some(&300_000) ); + let plan = plan_of(&physical, &solved, &outputs(&physical)).unwrap(); + let chain_node = plan + .nodes + .iter() + .find(|node| node.selected_physical_candidate == Some(PhysicalCandidateId(1))) + .unwrap(); + assert_eq!(chain_node.retention_ms, Some(300_000)); } /// Ga = `m offset 1s` and Gb = `m offset 2s`, each also computed from the @@ -261,27 +416,31 @@ fn acyclicity_forbids_two_groups_reading_each_other() { let b_from_a = try_candidate(&mut memo, b, shift(1_000, a), None).unwrap(); query(&mut memo, a, None); query(&mut memo, b, None); - let first = own_candidate; // v0 scan; v1 Ga①, v2 Gb①, v3 Ga②, v4 Gb②. let physical = physical( memo, &[ - (first(m), ONE_EVALUATION), - (first(a), ONE_EVALUATION), - (first(b), ONE_EVALUATION), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(a), ONE_EVALUATION), + (own_candidate(b), ONE_EVALUATION), (a_from_b, ONE_EVALUATION), (b_from_a, ONE_EVALUATION), ], ); - let costs = costs(&physical, &[0.0, 10.0, 10.0, 1.0, 1.0], &[]); - let solved = solve(&physical, &costs).unwrap(); + let (solved, _) = solve( + &physical, + &every_10s(2), + &[0.0, 10.0, 10.0, 1.0, 1.0], + &[], + &[], + ); + let solved = solved.unwrap(); + let selected = candidates(&solved); assert!( - solved.candidates == ids(&[0, 1, 4]) || solved.candidates == ids(&[0, 2, 3]), - "{:?}", - solved.candidates + selected == vec![0, 1, 4] || selected == vec![0, 2, 3], + "{selected:?}" ); - let outputs = vec![(QueryId(0), a), (QueryId(1), b)]; - plan_of(&physical, &[], &solved, &outputs).unwrap(); + plan_of(&physical, &solved, &outputs(&physical)).unwrap(); } /// x08's shape: `max by (job)(count_over_time(m[1m]))`. Path A (scan and @@ -330,7 +489,6 @@ fn a_candidate_dearer_than_a_feasible_plan_does_not_hide_a_cheaper_plan() { 1.666_666_666_666_666_7e-5, 2_553_517.822_748_626_6, ]; - let costs = costs(&physical, &w, &[]); - let solved = solve(&physical, &costs).unwrap(); - assert_eq!(solved.candidates, ids(&[3, 4, 5, 6, 7, 8])); + let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); + assert_eq!(candidates(&solved.unwrap()), vec![3, 4, 5, 6, 7, 8]); } diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index d1c641bd6..a4bf445c4 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -1,6 +1,6 @@ -//! An upper bound on the baseline MILP's optimum, and the candidates it +//! An upper bound on the selection MILP's optimum, and the executions it //! rules out: the cost of one feasible plan found greedily, and every -//! candidate that alone costs more than it. Leaving those out before the +//! execution that alone costs more than it. Leaving those out before the //! objective is scaled keeps a candidate costing millions per second from //! shrinking the differences between plans costing thousandths under the //! solver's tolerances. @@ -14,50 +14,59 @@ use std::collections::HashMap; -use asap_types::stages::PhysicalCandidateId; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::stages::Lifetime; -use super::model::{is_unmaterialized_query_time, plan_cost_per_second, SelectionInputs}; +use super::model::{selection_cost_per_second, QueryId, SelectionInputs}; +use crate::selectable::Execution; -/// How many times the greedy plan is rebuilt without a candidate that +/// How many times the greedy plan is rebuilt without an execution that /// made it infeasible before giving up on a bound. const REPAIRS: usize = 32; -/// Per (consumer, input), the costed edges into it, each as (producer, -/// $c_{u,c,i}$). -type EdgesInto = HashMap<(usize, usize), Vec<(usize, f64)>>; +/// Per (consumer execution, input), the costed edges into it, each as +/// (edge position, producer execution, $\rho_\kappa c_{\kappa,u,c,i}$). +type EdgesInto = HashMap<(usize, usize), Vec<(usize, usize, f64)>>; -/// Per input of a candidate, the edge its cheapest tree reads it over, as -/// (producer, $c_{u,c,i}$). -type TreeInputs = Vec>; +/// Per input of an execution, the edge its cheapest tree reads it over, as +/// (edge position, producer execution). +type TreeInputs = Vec>; -/// Per candidate, whether it costs more than a feasible plan -/// ([`feasible_plan_cost_per_second`]): its own $w_v$ plus the most its -/// input edges can take off ($\min(0, c_{u,v,i})$ per input) is over the -/// bound. Such a candidate is in no optimal plan, as long as every other -/// selected candidate costs at least 0 with its selected inputs. None is -/// ruled out when no feasible plan is found. +/// Per execution, by position in `ExecutionContexts::executions`, whether +/// it costs more than a feasible plan ([`feasible_plan_cost_per_second`]): +/// its own cost plus the most its input edges can take off ($\min(0, +/// c_{\kappa,u,v,i})$ per input) is over the bound. Such an execution is in +/// no optimal plan, as long as every other selected execution costs at +/// least 0 with its selected inputs. None is ruled out when no feasible +/// plan is found. pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec { - let count = inputs.physical.physical_candidates.len(); + let contexts = inputs.contexts; + let count = contexts.executions().len(); let Some(bound) = feasible_plan_cost_per_second(inputs) else { return vec![false; count]; }; - let mut cheapest_input_edge: HashMap<(PhysicalCandidateId, usize), f64> = HashMap::new(); - for (&edge, &cost) in &inputs.costs.edge_cost_change_per_second { - let edge = inputs.physical.edge(edge); - let key = (edge.consumer_candidate_id, edge.input_index_of_consumer); + let mut cheapest_input_edge: HashMap<(usize, usize), f64> = HashMap::new(); + for (&(consumer, edge, _), cost) in contexts.edges().iter().zip(&inputs.costs.per_edge) { + let Some(cost) = cost else { + continue; + }; + let key = ( + contexts.index_of(consumer), + inputs.physical.edge(edge).input_index_of_consumer, + ); let cheapest = cheapest_input_edge.entry(key).or_insert(0.0); - *cheapest = cheapest.min(cost); + *cheapest = cheapest.min(*cost); } let tolerance = 1e-9 * bound.abs().max(1e-12); (0..count) - .map(PhysicalCandidateId) - .map(|v| { - let Ok(own) = &inputs.costs.candidate_cost_per_second[v.0] else { + .map(|index| { + let Ok(own) = &inputs.costs.per_execution[index] else { return false; }; + let v = contexts.executions()[index].candidate; let least = own + (0..inputs.physical.input_count(v)) - .filter_map(|i| cheapest_input_edge.get(&(v, i))) + .filter_map(|i| cheapest_input_edge.get(&(index, i))) .sum::(); least > bound + tolerance }) @@ -65,41 +74,61 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec) -> Option { let physical = inputs.physical; - let count = physical.physical_candidates.len(); - // Each (consumer, input)'s costed edges: (edge, producer, c_{u,c,i}). + let contexts = inputs.contexts; + let count = contexts.executions().len(); let mut into: EdgesInto = HashMap::new(); - for (&id, &cost) in &inputs.costs.edge_cost_change_per_second { - let edge = physical.edge(id); - into.entry((edge.consumer_candidate_id.0, edge.input_index_of_consumer)) - .or_default() - .push((edge.producer_candidate_id.0, cost)); + for (position, (&(consumer, edge, producer), cost)) in contexts + .edges() + .iter() + .zip(&inputs.costs.per_edge) + .enumerate() + { + let Some(cost) = cost else { + continue; + }; + into.entry(( + contexts.index_of(consumer), + physical.edge(edge).input_index_of_consumer, + )) + .or_default() + .push((position, contexts.index_of(producer), *cost)); } - let inputs_of: Vec = (0..count) - .map(|v| physical.input_count(PhysicalCandidateId(v))) + let inputs_of: Vec = contexts + .executions() + .iter() + .map(|execution| physical.input_count(execution.candidate)) .collect(); + let unshared = |index: usize| { + let v = contexts.executions()[index].candidate; + let properties = physical.physical_candidates[v.0].physical_properties; + properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime + }; let mut excluded = vec![false; count]; for _ in 0..REPAIRS { let (tree_cost, chosen) = cheapest_trees(inputs, &into, &inputs_of, &excluded); - // Each query output's cheapest candidate, then the trees below. + // Each query output's cheapest execution, then the trees below. let mut uses = vec![0usize; count]; let mut outputs = Vec::new(); - for (_, candidates) in inputs.query_outputs { - let best = candidates + let mut answering: Vec<(QueryId, Execution)> = Vec::new(); + for (query, executions) in inputs.query_outputs { + let best = executions .iter() - .map(|v| v.0) - .filter(|&v| tree_cost[v].is_finite()) + .map(|&execution| contexts.index_of(execution)) + .filter(|&index| tree_cost[index].is_finite()) .min_by(|&a, &b| tree_cost[a].total_cmp(&tree_cost[b]))?; uses[best] += 1; outputs.push(best); + answering.push((*query, contexts.executions()[best])); } let plan = match plan_below(&outputs, &chosen, &inputs_of) { Ok(plan) => plan, @@ -108,31 +137,28 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { continue; } }; - for &(v, _) in &plan.edges { - uses[v] += 1; + for &(_, producer) in &plan.edges { + uses[producer] += 1; } if let Some(shared) = plan - .candidates + .executions .iter() .copied() - .find(|&v| is_unmaterialized_query_time(inputs, v) && uses[v] > 1) + .find(|&index| unshared(index) && uses[index] > 1) { excluded[shared] = true; continue; } - let cost = plan_cost_per_second( - inputs, - plan.candidates.iter().copied(), - plan.edges.iter().map(|&(_, cost)| cost), - ); + let edges: Vec = plan.edges.iter().map(|&(position, _)| position).collect(); + let (cost, _) = selection_cost_per_second(inputs, &plan.executions, &edges, &answering); return cost.is_finite().then_some(cost); } None } -/// Per candidate not `excluded`, the cost of its cheapest tree and, per +/// Per execution not `excluded`, the cost of its cheapest tree and, per /// input, the edge it reads it over: relaxed until no tree gets cheaper -/// (at most once per candidate), so a tree over a cycle of candidates is +/// (at most once per execution), so a tree over a cycle of executions is /// never cheaper than one without it when costs are not negative. fn cheapest_trees( inputs: &SelectionInputs<'_>, @@ -146,7 +172,7 @@ fn cheapest_trees( for _ in 0..count.max(1) { let mut changed = false; for v in 0..count { - let Ok(own) = &inputs.costs.candidate_cost_per_second[v] else { + let Ok(own) = &inputs.costs.per_execution[v] else { continue; }; if excluded[v] { @@ -159,13 +185,13 @@ fn cheapest_trees( .get(&(v, i)) .into_iter() .flatten() - .filter(|&&(u, _)| !excluded[u] && tree_cost[u].is_finite()) - .map(|&(u, c)| (u, c, c + tree_cost[u])) + .filter(|&&(_, u, _)| !excluded[u] && tree_cost[u].is_finite()) + .map(|&(position, u, c)| (position, u, c + tree_cost[u])) .min_by(|a, b| a.2.total_cmp(&b.2)); match cheapest { - Some((u, c, through)) => { + Some((position, u, through)) => { cost += through; - reads.push(Some((u, c))); + reads.push(Some((position, u))); } None => { cost = f64::INFINITY; @@ -188,13 +214,13 @@ fn cheapest_trees( /// A plan of the trees below some query outputs. struct GreedyPlan { - /// Its candidates, each once. - candidates: Vec, - /// Its edges, as (producer, $c_{u,c,i}$). - edges: Vec<(usize, f64)>, + /// Its executions, each once. + executions: Vec, + /// Its edges, as (edge position, producer execution). + edges: Vec<(usize, usize)>, } -/// The plan of the trees `chosen` below `outputs`; Err with a candidate +/// The plan of the trees `chosen` below `outputs`; Err with an execution /// on a cycle of them. fn plan_below( outputs: &[usize], @@ -204,7 +230,7 @@ fn plan_below( // 0 unvisited, 1 on the walk, 2 done. let mut state = vec![0u8; inputs_of.len()]; let mut plan = GreedyPlan { - candidates: Vec::new(), + executions: Vec::new(), edges: Vec::new(), }; for &output in outputs { @@ -216,12 +242,12 @@ fn plan_below( while let Some((v, i)) = stack.pop() { if i == inputs_of[v] { state[v] = 2; - plan.candidates.push(v); + plan.executions.push(v); continue; } stack.push((v, i + 1)); - let (u, c) = chosen[v][i].expect("a finite tree reads every input"); - plan.edges.push((u, c)); + let (position, u) = chosen[v][i].expect("a finite tree reads every input"); + plan.edges.push((position, u)); match state[u] { 0 => { state[u] = 1; diff --git a/crates/stage3-plan-selection/src/plan.rs b/crates/stage3-plan-selection/src/plan.rs index f955df74e..36f2552f1 100644 --- a/crates/stage3-plan-selection/src/plan.rs +++ b/crates/stage3-plan-selection/src/plan.rs @@ -1,8 +1,10 @@ -//! The plan Stage 3 builds from the baseline MILP's optimum as its last -//! step (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +//! The plan Stage 3 builds from the MILP's optimum as its last step +//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 3, "Plan output"): one `PhysicalASAPDAG` node per selected -//! candidate ([`candidate_node`], which costing builds too), its inputs -//! from the selected edges, each node's `selected_physical_candidate` set. +//! execution ([`candidate_node`], which costing builds too), its inputs +//! from the selected edges, each node's `selected_physical_candidate` set +//! and `retention_ms` its candidate's $h_v$. Several nodes may come from +//! one candidate, one per context it runs in. use std::collections::{BTreeMap, HashMap}; @@ -12,13 +14,10 @@ use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, }; use asap_types::plan::view_coverage; -use asap_types::stages::{ - Lifetime, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, -}; -use asap_types::workload::QueryWorkloadEntry; +use asap_types::stages::{Lifetime, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId}; use crate::milp::model::{QueryId, SolvedSelection}; -use crate::retention::retention_need_ms; +use crate::selectable::Execution; /// Candidate `v` as a plan node with id `id`, its input `i` read from node /// `input_ids[i]`: its operator, timing, lifetime and θ, its group's schema @@ -60,97 +59,70 @@ pub(crate) fn candidate_node( /// The plan of `solved`, whose output `j` answers `outputs[j]` (a query and /// the group its root, or one operator its scalar root, reads). Nodes are -/// numbered producers first. A node that retains rows (materialized, or a -/// query-time `Scan`) has `retention_ms` the most its selected consumers -/// read back ([`retention_need_ms`]), and at least the range-evaluation -/// span of a query it answers directly. +/// numbered producers first. A node whose candidate has $h_v$ retains that +/// much (`retention_ms`); one that retains rows a consumer reads unbounded +/// has `retention_ms` `None` and keeps every row. pub(crate) fn plan_of( physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], solved: &SolvedSelection, outputs: &[(QueryId, MemoGroupId)], ) -> Result { - let mut producer_of: HashMap<(PhysicalCandidateId, usize), PhysicalCandidateId> = - HashMap::new(); - let mut consumers_of: BTreeMap> = BTreeMap::new(); - for &id in &solved.edges { - let edge = physical.edge(id); + let mut producer_of: HashMap<(Execution, usize), Execution> = HashMap::new(); + let mut consumers_of: BTreeMap> = BTreeMap::new(); + for &(consumer, edge, producer) in &solved.edges { producer_of.insert( - (edge.consumer_candidate_id, edge.input_index_of_consumer), - edge.producer_candidate_id, + (consumer, physical.edge(edge).input_index_of_consumer), + producer, ); - consumers_of - .entry(edge.producer_candidate_id) - .or_default() - .push(id); + consumers_of.entry(producer).or_default().push(consumer); } - let consumers = |v: PhysicalCandidateId| consumers_of.get(&v).into_iter().flatten().copied(); - let readers_of = - |v: PhysicalCandidateId| consumers(v).map(|edge| physical.edge(edge).consumer_candidate_id); - let inputs = |v: PhysicalCandidateId| -> Vec { - (0..physical.input_count(v)) - .map(|i| producer_of[&(v, i)]) - .collect() - }; + let input_count = |execution: Execution| physical.input_count(execution.candidate); // Producers first: Kahn's order over the selected edges, which (E) // keeps acyclic. - let mut waiting: HashMap = solved - .candidates + let mut waiting: HashMap = solved + .executions .iter() - .map(|&v| (v, physical.input_count(v))) + .map(|&execution| (execution, input_count(execution))) .collect(); - let mut ready: Vec = solved - .candidates + let mut ready: Vec = solved + .executions .iter() .rev() .copied() - .filter(|v| waiting[v] == 0) + .filter(|execution| waiting[execution] == 0) .collect(); - let mut node_of: HashMap = HashMap::new(); - let mut order = Vec::with_capacity(solved.candidates.len()); - while let Some(v) = ready.pop() { - node_of.insert(v, order.len()); - order.push(v); - for edge in consumers(v) { - let consumer = physical.edge(edge).consumer_candidate_id; - let left = waiting.get_mut(&consumer).expect("a selected consumer"); + let mut node_of: HashMap = HashMap::new(); + let mut order = Vec::with_capacity(solved.executions.len()); + while let Some(execution) = ready.pop() { + node_of.insert(execution, order.len()); + order.push(execution); + for consumer in consumers_of.get(&execution).into_iter().flatten() { + let left = waiting.get_mut(consumer).expect("a selected consumer"); *left -= 1; if *left == 0 { - ready.push(consumer); + ready.push(*consumer); } } } - if order.len() != solved.candidates.len() { + if order.len() != solved.executions.len() { return Err(PhysicalASAPDAGValidationError::Cycle); } - // The span of the queries each candidate answers directly. - let mut output_span_ms: HashMap = HashMap::new(); - for (&(query, _), &v) in outputs.iter().zip(&solved.outputs) { - let span_ms = entries - .get(query.0) - .and_then(|entry| entry.range_evaluation) - .map_or(0, |range| range.span_ms); - let known = output_span_ms.entry(v).or_default(); - *known = (*known).max(span_ms); - } let nodes = order .iter() .enumerate() - .map(|(id, &v)| { - let input_ids: Vec = - inputs(v).iter().map(|producer| node_of[producer]).collect(); - let mut node = candidate_node(physical, v, id, &input_ids); - if node.retains_rows() { - node.retention_ms = retention_need_ms(physical, entries, v, readers_of) - .map(|need_ms| need_ms.max(output_span_ms.get(&v).copied().unwrap_or(0))); - } + .map(|(id, &execution)| { + let input_ids: Vec = (0..input_count(execution)) + .map(|i| node_of[&producer_of[&(execution, i)]]) + .collect(); + let mut node = candidate_node(physical, execution.candidate, id, &input_ids); + node.retention_ms = solved.retention_ms.get(&execution.candidate).copied(); node }) .collect(); let output_node: HashMap<(QueryId, MemoGroupId), PhysicalASAPNodeId> = outputs .iter() .zip(&solved.outputs) - .map(|(&output, v)| (output, node_of[v])) + .map(|(&output, execution)| (output, node_of[execution])) .collect(); let mut roots = Vec::new(); let mut scalar_roots = Vec::new(); diff --git a/crates/stage3-plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs index 31768b6af..e999389ad 100644 --- a/crates/stage3-plan-selection/src/report.rs +++ b/crates/stage3-plan-selection/src/report.rs @@ -13,7 +13,7 @@ use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidate, PrunedCandidate}; use thiserror::Error; -use crate::cost::SelectionCosts; +use crate::cost::ExecutionCosts; /// Peak memory above which a candidate's report carries a warning when the /// deployment declares no memory capacity: 10^12 bytes (1 TB), more than @@ -193,12 +193,12 @@ pub enum ErrorSelectingPlan { UnsupportedCostModel(String), } -/// The selection MILP is infeasible, with the reasons candidates could not +/// The selection MILP is infeasible, with the reasons executions could not /// be costed. -pub(crate) fn infeasible(costs: &SelectionCosts) -> ErrorSelectingPlan { +pub(crate) fn infeasible(costs: &ExecutionCosts) -> ErrorSelectingPlan { let mut reasons: Vec<&str> = Vec::new(); for reason in costs - .candidate_cost_per_second + .per_execution .iter() .filter_map(|cost| cost.as_ref().err()) { @@ -207,11 +207,12 @@ pub(crate) fn infeasible(costs: &SelectionCosts) -> ErrorSelectingPlan { } } let mut reason = "no plan gives every query output a candidate whose every input has an \ - admissible producer (the selection MILP is infeasible)" + admissible producer that delivers on its context (the selection MILP is \ + infeasible)" .to_string(); if !reasons.is_empty() { reason += &format!( - "; candidates left out because they could not be costed: {}", + "; executions left out because they could not be costed: {}", reasons.join("; ") ); } @@ -296,35 +297,4 @@ fn sketches_below( } #[cfg(test)] -mod tests { - use super::*; - use asap_cost::{ClusterLimits, ClusterObjective, ResourceUnitCost}; - - /// Without a declared memory capacity nothing rejects a candidate for - /// its memory, so a peak over `LARGE_PEAK_MEMORY_BYTES` is reported - /// as a warning (the eval's 24 h quantiles plan peaked at 1.43 TB). - /// Under cost model 3 the cluster's memory is declared and its capacity - /// check applies instead, so there is no warning. - #[test] - fn a_very_large_peak_memory_is_warned_about_only_without_a_declared_capacity() { - let usage_over_time = CostModel::UsageOverTime(ResourceUnitCost::illustrative()); - assert_eq!( - large_peak_memory_warning(1_430_000_000_000, &usage_over_time).as_deref(), - Some( - "peak memory 1430000000000 bytes is over 1000000000000 bytes and the deployment \ - declares no memory capacity to check it against" - ) - ); - assert_eq!( - large_peak_memory_warning(LARGE_PEAK_MEMORY_BYTES, &usage_over_time), - None - ); - let cluster = CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective: ClusterObjective::LowestLatency, - }); - assert_eq!(large_peak_memory_warning(1_430_000_000_000, &cluster), None); - } -} +mod tests; diff --git a/crates/stage3-plan-selection/src/report/tests.rs b/crates/stage3-plan-selection/src/report/tests.rs new file mode 100644 index 000000000..c73f67f71 --- /dev/null +++ b/crates/stage3-plan-selection/src/report/tests.rs @@ -0,0 +1,30 @@ +use super::*; +use asap_cost::{ClusterLimits, ClusterObjective, ResourceUnitCost}; + +/// Without a declared memory capacity nothing rejects a candidate for +/// its memory, so a peak over `LARGE_PEAK_MEMORY_BYTES` is reported +/// as a warning (the eval's 24 h quantiles plan peaked at 1.43 TB). +/// Under cost model 3 the cluster's memory is declared and its capacity +/// check applies instead, so there is no warning. +#[test] +fn a_very_large_peak_memory_is_warned_about_only_without_a_declared_capacity() { + let usage_over_time = CostModel::UsageOverTime(ResourceUnitCost::illustrative()); + assert_eq!( + large_peak_memory_warning(1_430_000_000_000, &usage_over_time).as_deref(), + Some( + "peak memory 1430000000000 bytes is over 1000000000000 bytes and the deployment \ + declares no memory capacity to check it against" + ) + ); + assert_eq!( + large_peak_memory_warning(LARGE_PEAK_MEMORY_BYTES, &usage_over_time), + None + ); + let cluster = CostModel::ClusterCapacity(ClusterLimits { + memory_bytes: u64::MAX, + disk_bytes: u64::MAX, + cpu_cores: None, + objective: ClusterObjective::LowestLatency, + }); + assert_eq!(large_peak_memory_warning(1_430_000_000_000, &cluster), None); +} diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 0e19a2bd6..f399aaaea 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -1,20 +1,75 @@ //! How far back from the evaluation time a candidate that retains rows (a -//! materialized one, or a query-time `Scan`) must keep them, and how far -//! back one execution of a query-time `Scan` reads, for the candidates -//! that may read it: [`asap_cost::read_back_ms`] over candidates +//! materialized one, or a query-time `Scan`) must keep them for a reader +//! over one edge, $\operatorname{need}_{\kappa,u,c,i}$ of constraint (R), +//! or for a query reading it directly, and how far back one execution of a +//! query-time `Scan` reads for that reader: [`asap_cost::read_back_ms`] +//! over candidates //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 5, "Cost and retention"). use asap_types::ir::{NonASAPOp, Operator}; use asap_types::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; -use asap_types::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId}; +use asap_types::stages::{ + MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, +}; use asap_types::workload::QueryWorkloadEntry; +/// $\operatorname{need}_{\kappa,u,c,i}$: how far back from the evaluation +/// time `edge`'s producer $u$ must keep its rows for its consumer $c$ +/// ([`retention_need_ms`] with $c$ $u$'s only reader, and every candidate +/// that may read $c$ above it); `None` when read unbounded. +/// +/// Example: `TimeRange(5m)` over a chain of 10 s windows, read by a range +/// query with a 1 h span: 3,900,000 ms. +pub(crate) fn edge_retention_need_ms( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + edge: PhysicalEdgeId, +) -> Option { + let producer = physical.edge(edge).producer_candidate_id; + retention_need_ms(physical, entries, producer, readers_through(physical, edge)) +} + +/// How far back one execution of the query-time `Scan` that is `edge`'s +/// producer reads for its consumer ([`scan_read_ms`] with that consumer +/// its only reader); `None` when read whole. +pub(crate) fn edge_scan_read_ms( + physical: &PhysicalASAPMemoCandidates, + edge: PhysicalEdgeId, +) -> Option { + let producer = physical.edge(edge).producer_candidate_id; + scan_read_ms(physical, producer, readers_through(physical, edge)) +} + +/// How far back a query reads the output it reads directly: its +/// range-evaluation span (0 without one). +pub(crate) fn output_retention_need_ms(entry: &QueryWorkloadEntry) -> u64 { + entry.range_evaluation.map_or(0, |range| range.span_ms) +} + +/// The readers of each candidate when `edge`'s producer is read over +/// `edge` only: its consumer, and above it every candidate that may read +/// one (its admissible edges out). +fn readers_through( + physical: &PhysicalASAPMemoCandidates, + edge: PhysicalEdgeId, +) -> impl Fn(PhysicalCandidateId) -> Vec + '_ { + let edge = physical.edge(edge); + move |v| match v == edge.producer_candidate_id { + true => vec![edge.consumer_candidate_id], + false => physical + .admissible_edges_out_of(v) + .iter() + .map(|&out| physical.edge(out).consumer_candidate_id) + .collect(), + } +} + /// How far back from the evaluation time `producer`'s rows must be kept /// for its readers (`readers_of`, the candidates that may read it): /// [`asap_cost::read_back_ms`] with, at each reader, the longest /// range-evaluation span of the queries reaching it. -pub(crate) fn retention_need_ms>( +fn retention_need_ms>( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], producer: PhysicalCandidateId, @@ -37,7 +92,7 @@ pub(crate) fn retention_need_ms>( /// How far back from the evaluation time one execution of the query-time /// `Scan` `producer` reads, for its readers (`readers_of`): /// [`asap_cost::read_back_ms`] without a range-evaluation span. -pub(crate) fn scan_read_ms>( +fn scan_read_ms>( physical: &PhysicalASAPMemoCandidates, producer: PhysicalCandidateId, readers_of: impl Fn(PhysicalCandidateId) -> R, diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index f97f30726..b8158261d 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -1,37 +1,279 @@ -//! What the baseline selection may select: the selectable edges (each -//! admissible edge whose producer delivers on every evaluation its -//! consumer serves) and, per query output, the candidates that may answer -//! it. +//! What the selection may select +//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +//! section 5, "Execution contexts, by example"): the distinct execution +//! contexts $\kappa$ of a workload, every execution of a candidate in a +//! context, the edges $y_{\kappa,u,c,i}$ between executions, and per query +//! output the executions that may answer it ([`ExecutionContexts::new`]). +//! +//! - A query is evaluated in the evaluation context of its grid +//! ([`evaluation_grid`]); queries on one grid share it, a range query's +//! steps included. +//! - A candidate with $\ell$ = None or `OneEvaluation` runs once per context +//! that selects it ($x_{\kappa,v}$), the context of its consumer; at +//! ingestion time, where each arriving row is computed once whatever +//! reads it (`asap_cost::computations`), in the one ingestion context +//! [`EVERY_ARRIVING_ROW`]. +//! - A candidate with $\ell$ = `AcrossEvaluations` runs once, in its own +//! context `Materialized(v)` ($x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$). +//! Its inputs run in that context at query time; at ingestion time its +//! producers must deliver on the ingestion pipeline of its refresh +//! window. +//! - The input of a `TimeShift` anchored with `@ end()` or `@ start()` runs +//! on the grid of the request end (start) times: once per request of a +//! range query, as an instant query with its recurrence is evaluated. +//! - An input of a consumer in $\kappa$ reads a producer, over an admissible +//! edge, only where the producer delivers on $\kappa$ +//! (`PhysicalCandidate::delivers_on`). use std::collections::BTreeSet; +use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::QueryRoot; +use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; +use asap_types::plan::materialized_view::RefreshWindow; use asap_types::stages::{ output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, - MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, + ExecutionContextId, Lifetime, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, }; -use asap_types::workload::QueryWorkloadEntry; +use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; -use crate::cost::SelectionCosts; use crate::milp::model::QueryId; -/// The edges a plan may select: admissible, with a producer that delivers -/// on every evaluation its query-time consumer serves -/// ([`delivers_to_consumer`]). -pub(crate) fn selectable_edges( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], -) -> Vec { - physical - .edges() - .filter(|(_, edge)| { - edge.is_logically_and_physically_admissible() - && delivers_to_consumer(physical, entries, edge) +/// The one context of every ingestion-time execution that is not +/// materialized: each second of arriving rows, which it computes once for +/// every ingestion-time view of every refresh window that reads it. +pub(crate) const EVERY_ARRIVING_ROW: ExecutionContext = + ExecutionContext::IngestionPipeline(RefreshWindow { + width_ms: 1_000, + phase_ms: 0, + }); + +/// One run of candidate `candidate` in context `context`: a node of the +/// plan. $x^{\mathrm{mat}}_v$ is the execution of $v$ in +/// `Materialized(v)`, $x_{\kappa,v}$ that of $v$ in $\kappa$. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct Execution { + pub(crate) context: ExecutionContextId, + pub(crate) candidate: PhysicalCandidateId, +} + +/// The distinct contexts of a workload and every execution and edge the +/// selection may choose ([`Self::new`]). +pub(crate) struct ExecutionContexts { + /// The distinct contexts, by [`ExecutionContextId`]. + contexts: Vec, + /// Every execution reached from a query output, ascending. + executions: Vec, + /// Every $y_{\kappa,u,c,i}$ as (consumer execution, edge $(u,c,i)$, + /// producer execution), ascending; $\kappa$ is the consumer's context. + edges: Vec<(Execution, PhysicalEdgeId, Execution)>, +} + +impl ExecutionContexts { + /// The contexts, executions and edges of `physical` for the queries + /// `entries` (by query index), reached from `outputs` (each query output + /// reference, a query and the group it reads, [`output_groups`]), and + /// per output the executions that may answer it: a candidate of its + /// group a query-time consumer may read + /// (`producer_satisfies_physical_requirements`) that delivers on the + /// query's context. + pub(crate) fn new( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + outputs: &[(QueryId, MemoGroupId)], + ) -> (Self, Vec<(QueryId, Vec)>) { + let mut contexts = Self { + contexts: Vec::new(), + executions: Vec::new(), + edges: Vec::new(), + }; + let mut reached: BTreeSet = BTreeSet::new(); + let mut pending: Vec = Vec::new(); + let mut output_executions = Vec::with_capacity(outputs.len()); + for &(query, group) in outputs { + let context = contexts.context_id(ExecutionContext::Evaluation { + grid: evaluation_grid(&entries[query.0]), + }); + let mut answering = Vec::new(); + for (v, candidate) in physical.physical_candidates.iter().enumerate() { + if candidate.logical_candidate_id.group != group + || !producer_satisfies_physical_requirements( + candidate, + ExecutionTiming::QueryTime, + ) + { + continue; + } + if let Some(execution) = + contexts.execution_reading(physical, context, PhysicalCandidateId(v)) + { + answering.push(execution); + if reached.insert(execution) { + pending.push(execution); + } + } + } + output_executions.push((query, answering)); + } + let mut edges = BTreeSet::new(); + while let Some(consumer) = pending.pop() { + let c = consumer.candidate; + let context = contexts.input_context(physical, entries, consumer); + for input in 0..physical.input_count(c) { + for &edge in physical.admissible_edges_into(c, input) { + let u = physical.edge(edge).producer_candidate_id; + if let Some(producer) = contexts.execution_reading(physical, context, u) { + edges.insert((consumer, edge, producer)); + if reached.insert(producer) { + pending.push(producer); + } + } + } + } + } + contexts.executions = reached.into_iter().collect(); + contexts.edges = edges.into_iter().collect(); + (contexts, output_executions) + } + + /// The id of `context`, added to the list when new. + fn context_id(&mut self, context: ExecutionContext) -> ExecutionContextId { + match self.contexts.iter().position(|known| *known == context) { + Some(index) => ExecutionContextId(index), + None => { + self.contexts.push(context); + ExecutionContextId(self.contexts.len() - 1) + } + } + } + + /// The execution of producer `u` a consumer whose inputs run in + /// `context` reads, when `u` delivers on it: its own for an + /// `AcrossEvaluations` producer, the one ingestion context + /// [`EVERY_ARRIVING_ROW`] for another ingestion-time producer, otherwise + /// its run in `context`. + fn execution_reading( + &mut self, + physical: &PhysicalASAPMemoCandidates, + context: ExecutionContextId, + u: PhysicalCandidateId, + ) -> Option { + let producer = &physical.physical_candidates[u.0]; + if !producer.delivers_on(physical, &self.contexts[context.0]) { + return None; + } + let context = match ( + producer.physical_properties.lifetime(), + producer.physical_properties.timing(), + ) { + (Lifetime::AcrossEvaluations(_), _) => { + self.context_id(ExecutionContext::Materialized(u)) + } + (_, ExecutionTiming::IngestionTime) => self.context_id(EVERY_ARRIVING_ROW), + (Lifetime::None | Lifetime::OneEvaluation(_), ExecutionTiming::QueryTime) => context, + }; + Some(Execution { + context, + candidate: u, }) - .map(|(id, _)| id) - .collect() + } + + /// The context the inputs of `consumer` run in: an `AcrossEvaluations` + /// candidate's at ingestion time, the ingestion pipeline of its refresh + /// window (its chain width, else its group's evaluation grid); + /// a `TimeShift` anchored at `@ end()` or `@ start()` in an evaluation + /// context, the grid of its queries' request end or start times; else + /// the consumer's own context. + fn input_context( + &mut self, + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + consumer: Execution, + ) -> ExecutionContextId { + let candidate = &physical.physical_candidates[consumer.candidate.0]; + let memo = &physical.logical_candidates.memo; + let group = candidate.logical_candidate_id.group; + let at_ingestion = candidate.physical_properties.timing() == ExecutionTiming::IngestionTime; + if at_ingestion + && matches!( + candidate.physical_properties.lifetime(), + Lifetime::AcrossEvaluations(_) + ) + { + let refresh_window = match candidate.chain_tumbling_window_width_ms(physical) { + Some(width_ms) => Some(RefreshWindow { + width_ms, + phase_ms: 0, + }), + None => physical.logical_candidates.evaluation_grid_of_group[group.0 as usize].map( + |grid| RefreshWindow { + width_ms: grid.period_ms, + phase_ms: grid.phase_ms, + }, + ), + }; + return match refresh_window { + Some(window) => self.context_id(ExecutionContext::IngestionPipeline(window)), + None => consumer.context, + }; + } + let at = match memo + .candidate(candidate.logical_candidate_id) + .expression + .operator + { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at, + _ => None, + }; + let ( + Some(at @ (AtModifier::End | AtModifier::Start)), + ExecutionContext::Evaluation { grid }, + ) = (at, self.contexts[consumer.context.0]) + else { + return consumer.context; + }; + let period_ms = memo.groups()[group.0 as usize] + .reached_by_queries() + .iter() + .filter_map(|&query| entries.get(query)) + .filter(|entry| evaluation_grid(entry) == grid) + .fold(0, |period_ms, entry| { + greatest_common_divisor(period_ms, request_time_grid_ms(entry, at)) + }); + match period_ms { + 0 => consumer.context, + period_ms => self.context_id(ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms: 0, + }, + }), + } + } + + /// The context with id `id`. + pub(crate) fn context(&self, id: ExecutionContextId) -> ExecutionContext { + self.contexts[id.0] + } + + /// Every execution reached from a query output, ascending. + pub(crate) fn executions(&self) -> &[Execution] { + &self.executions + } + + /// Every $y_{\kappa,u,c,i}$ as (consumer execution, edge, producer + /// execution), ascending. + pub(crate) fn edges(&self) -> &[(Execution, PhysicalEdgeId, Execution)] { + &self.edges + } + + /// The position of `execution` in [`Self::executions`]. + pub(crate) fn index_of(&self, execution: Execution) -> usize { + self.executions + .binary_search(&execution) + .expect("a reached execution") + } } /// Each query output reference (a query, and the group its operator root, @@ -52,72 +294,28 @@ pub(crate) fn output_groups(physical: &PhysicalASAPMemoCandidates) -> Vec<(Query .collect() } -/// Per output of `outputs`, the candidates of its group that may answer -/// it: costed, readable at query time and delivering on its query's -/// evaluations. -pub(crate) fn query_outputs( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - costs: &SelectionCosts, - outputs: &[(QueryId, MemoGroupId)], -) -> Vec<(QueryId, Vec)> { - outputs - .iter() - .map(|&(query, group)| { - let candidates = physical - .physical_candidates - .iter() - .enumerate() - .filter(|(v, candidate)| { - candidate.logical_candidate_id.group == group - && costs.candidate_cost_per_second[*v].is_ok() - && producer_satisfies_physical_requirements( - candidate, - ExecutionTiming::QueryTime, - ) - && candidate.delivers_on(physical, &evaluation_context(&entries[query.0])) - }) - .map(|(v, _)| PhysicalCandidateId(v)) - .collect(); - (query, candidates) - }) - .collect() -} - -/// The evaluation context of `entry`'s query: the grid its evaluations -/// fall on (`QueryRecurrence::evaluation_grid_ms`), anchored at the epoch. -/// Times that are not known may fall on any millisecond: a 1 ms grid, on -/// which only producers stored at every time deliver. -fn evaluation_context(entry: &QueryWorkloadEntry) -> ExecutionContext { - ExecutionContext::Evaluation { - grid: EvaluationGrid { - period_ms: entry - .recurrence - .evaluation_grid_ms(entry.range_evaluation) - .unwrap_or(1), - phase_ms: 0, - }, +/// The evaluation grid of `entry`'s query, the grid of its context: the +/// times its evaluations fall on (`QueryRecurrence::evaluation_grid_ms` of +/// its range evaluation), anchored at the epoch. Times that are not known +/// may fall on any millisecond: a 1 ms grid, on which only producers stored +/// at every time deliver. +pub(crate) fn evaluation_grid(entry: &QueryWorkloadEntry) -> EvaluationGrid { + EvaluationGrid { + period_ms: entry + .recurrence + .evaluation_grid_ms(entry.range_evaluation) + .unwrap_or(1), + phase_ms: 0, } } -/// Whether `edge`'s producer delivers on every evaluation its query-time -/// consumer serves: on the evaluation context of each query reaching the -/// consumer's group ([`PhysicalCandidate::delivers_on`]). The baseline has -/// no other context: an ingestion-time consumer's (its pipeline) is checked -/// with execution contexts. -fn delivers_to_consumer( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - edge: &PhysicalEdge, -) -> bool { - let producer: &PhysicalCandidate = &physical.physical_candidates[edge.producer_candidate_id.0]; - let consumer = &physical.physical_candidates[edge.consumer_candidate_id.0]; - if consumer.physical_properties.timing() != ExecutionTiming::QueryTime { - return true; - } - physical.logical_candidates.memo.groups()[consumer.logical_candidate_id.group.0 as usize] - .reached_by_queries() - .iter() - .filter_map(|&query| entries.get(query)) - .all(|entry| producer.delivers_on(physical, &evaluation_context(entry))) +/// The grid, in ms, of the times `entry`'s requests anchor `@ end()` or +/// `@ start()` (`at`) at +/// ([`asap_cost::plan::execution_demand::request_time_grid_ms`]). +pub(crate) fn request_time_grid_ms(entry: &QueryWorkloadEntry, at: AtModifier) -> u64 { + asap_cost::plan::execution_demand::request_time_grid_ms( + &entry.recurrence, + entry.range_evaluation, + at, + ) } diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 96debd9e1..02ebeb049 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -3,8 +3,8 @@ //! with a time index. A chain has no range and no window count: a //! `TimeRange` (and `TimeShift`s) over it select the windows a query reads, //! a `SummaryMerge` over that selection is the state of the chain's build -//! over the same range, and how many windows it retains is $h_v$ (not yet -//! specified). +//! over the same range, and how many windows it retains is $h_v$, the +//! longest its selected consumers read back (Stage 3's constraint (R)). use super::asap::ASAPOp; use super::input::OperatorInput; use super::node::Operator; diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index 8c56560c7..6b44df160 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -49,6 +49,11 @@ impl PhysicalCandidate { /// | at query time over arriving rows, a window ending at E (`sum_over_time(x[5m] @ T)`, `ts BETWEEN a AND b`) | its own grid: final only from E on, and a context carries no absolute time to compare with E | /// /// `DataArrival::Unknown` counts as arriving. + /// The times of a + /// `Materialized(v)` context are $v$'s own: every window end of a chain + /// $v$ (its width at phase 0), else its group's evaluation grid; with + /// neither they are not known, and no time-dependent producer delivers + /// on it. pub fn delivers_on( &self, candidates: &PhysicalASAPMemoCandidates, @@ -60,24 +65,14 @@ impl PhysicalCandidate { Lifetime::AcrossEvaluations(_) => { let logical = &candidates.logical_candidates; let group = self.logical_candidate_id.group; - let chain_width_ms = match logical - .memo - .candidate(self.logical_candidate_id) - .expression - .operator - { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(tumbling_window_width_ms), - _ => None, - }; + let chain_tumbling_window_width_ms = + self.chain_tumbling_window_width_ms(candidates); let group_properties = &logical.memo.groups()[group.0 as usize].group_properties; let final_at_every_time = !group_properties.depends_on_evaluation_time && (self.physical_properties.timing() == ExecutionTiming::IngestionTime || logical.memo.data_arrival() == DataArrival::AtRest || group_properties.scanned_sources.is_empty()); - if chain_width_ms.is_none() && final_at_every_time { + if chain_tumbling_window_width_ms.is_none() && final_at_every_time { return true; } let (period_ms, phase_ms) = match context { @@ -85,13 +80,20 @@ impl PhysicalCandidate { ExecutionContext::IngestionPipeline(window) => { (window.width_ms, window.phase_ms) } - ExecutionContext::Materialized(_) => { - todo!( - "#1207 PR 7: the times of a Materialized(v) context, v's own schedule" - ) + ExecutionContext::Materialized(v) => { + let materialized = &candidates.physical_candidates[v.0]; + match materialized.chain_tumbling_window_width_ms(candidates) { + Some(width_ms) => (width_ms, 0), + None => match logical.evaluation_grid_of_group + [materialized.logical_candidate_id.group.0 as usize] + { + Some(grid) => (grid.period_ms, grid.phase_ms), + None => return false, + }, + } } }; - match chain_width_ms { + match chain_tumbling_window_width_ms { Some(width_ms) => { period_ms.is_multiple_of(width_ms) && phase_ms.is_multiple_of(width_ms) } @@ -105,6 +107,27 @@ impl PhysicalCandidate { } } } + + /// Its tumbling-window width when it is a chain + /// (`SummaryTumblingWindowChain`). + pub fn chain_tumbling_window_width_ms( + &self, + candidates: &PhysicalASAPMemoCandidates, + ) -> Option { + match candidates + .logical_candidates + .memo + .candidate(self.logical_candidate_id) + .expression + .operator + { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(tumbling_window_width_ms), + _ => None, + } + } } /// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 2685a3d28..8f8775588 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -472,6 +472,70 @@ fn a_window_pinned_by_at_t_delivers_on_every_context_once_nothing_changes_it() { } } +/// In `Materialized(v)` a producer delivers on $v$'s own times: a +/// 5 s chain inside the computation of a 10 s chain (every 10 s) and of +/// X = the scan's last 5 minutes stored on its 60 s grid, but not a +/// 10 s chain inside X's computation every 25 s, nor anything +/// time-dependent inside a build with neither a width nor a grid. +#[test] +fn delivers_on_the_own_times_of_a_materialized_computation() { + let grid = |period_ms| { + Some(EvaluationGrid { + period_ms, + phase_ms: 0, + }) + }; + let materialized = |grid_of_x| { + let mut candidates = candidates(memo(), vec![None, None, None, None, grid_of_x]); + candidates.physical_candidates = vec![ + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + candidate(BUILD, QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + ]; + candidates + }; + let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + let across_10s = candidate( + WINDOWS_10S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + let in_computation_of = |v| ExecutionContext::Materialized(PhysicalCandidateId(v)); + for (candidates, producer, context, delivers) in [ + ( + materialized(grid(60_000)), + &across_5s, + in_computation_of(0), + true, + ), + ( + materialized(grid(60_000)), + &across_5s, + in_computation_of(1), + true, + ), + ( + materialized(grid(25_000)), + &across_10s, + in_computation_of(1), + false, + ), + ( + materialized(grid(60_000)), + &across_5s, + in_computation_of(2), + false, + ), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} in {context:?}", + producer.logical_candidate_id + ); + } +} + /// `producer_satisfies_physical_requirements` over every valid producer state and every valid /// consumer state: a query-time producer serves only query-time /// consumers, an ingestion-time producer that is not materialized only diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 0ddf66196..30e0db06d 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -479,10 +479,17 @@ then the Pass 2 transformations), and expand shared sizes, tumbling-window horiz (planned) raw windows into concrete groups, so that choosing a threshold is choosing a group; choose one candidate per needed group; compose the choice into one plan and check and cost it with Stage 2 and Stage 3. The search is -the group selection MILP (mixed-integer linear program, #1095): it chooses one -candidate per memo group and execution context, with each shareable candidate -charged once, and is solved once; it prices cost model 1 only (cost models 2 -and 3 are an `UnsupportedCostModel` error, #888). Stage 3 builds the chosen +the selection MILP (mixed-integer linear program, concepts doc +sections 3 and 5) over execution contexts: a candidate with lifetime None or +`OneEvaluation` runs once per context that selects it ($x_{\kappa,v}$; a +context is an evaluation grid shared by the queries on it, an ingestion +pipeline's refresh window, or a materialized candidate's own computation), an +`AcrossEvaluations` candidate once ($x^{\mathrm{mat}}_v$), every edge per +context of its consumer ($y_{\kappa,u,c,i}$, only where `delivers_on` +holds), and a candidate that retains time-indexed rows retains what its +selected consumers read back ($h_v$, constraint (R)). It is solved once and +costs under cost model 1 only (cost models 2 and 3 are an +`UnsupportedCostModel` error). Stage 3 builds the chosen plan, a `PhysicalASAPDAG`, as its last step, so it does not depend on the Stage 2 crate. The facade runs Pass 2, Stage 2 and Stage 3 once, over the full tumbling-window-width domain, with no time limit. @@ -496,7 +503,7 @@ exactly come from `asap_accuracy::ErrorMetricsOfMemo`): the crate root's `generate_logical_asap_memo_candidates` runs them); the memo types Stage 1 hands Stage 2 are in `crates/types/src/stages/` (`asap_types::stages`); the facade that runs the stages is in `crates/planner/src/run.rs`; the search over Stage 2's -candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `groups/`, `milp/`). +candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `selectable.rs`, `cost.rs`, `retention.rs`, `milp/`, `plan.rs`, `report.rs`). | Step | Entry point | Path | |---|---|---| @@ -504,7 +511,7 @@ candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `groups/`, `milp | Pass 1 | `pass1_memo` → `Memo`, the one entry: inside it, the computation targets (walking the memo groups, less the summaries the deployment does not support; `ComputationTarget` is Pass 1's private working type), then each target's ways to compute added as candidates of the groups, a summary with one build per configuration of its `parameter_list` | `stage1-pass1-per-computation-candidates/src/lib.rs`; `computation_targets.rs`; `candidates/` (`mod.rs`, `pending.rs`, `measure_output.rs`); `summary_realization.rs` builds the summary inputs (the sketch algorithms that serve an intent, each row's update) | | Memo types | `Memo`, `MemoGroup`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges; then each group's evaluation grid and the groups each query may read only exactly | `stage1-pass2-reuse-candidates/src/` (`lib.rs`, `rules/{candidates.rs,rollup/,topk.rs,chains.rs}`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, `evaluation_grids.rs`; the readout levels: `asap_accuracy::readout_levels`; the groups each query may read only exactly: `asap_accuracy::ErrorMetricsOfMemo::exact_only_groups_of_query`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | -| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | +| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `execution_contexts.rs` for $h_v$ and (R), `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -1273,11 +1280,9 @@ not check executor support. It then removes the candidates no buildable input or query output can use (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`), repeating until nothing more is removed. Stage 2 makes no execution-scope -copies: how many times a candidate runs is Stage 3's choice, and a -selection names each run as a node of its `DagOfSelectedCandidates`. -Today's Stage 3 engine, whose MILP runs each candidate at most once, makes -the copies it needs itself before solving and decodes each selected copy -into a node. +copies: how many times a candidate runs is Stage 3's choice. Stage 3 runs +a candidate with lifetime None or `OneEvaluation` once per execution +context that selects it, and builds each run as a node of the plan. *Example for Algorithm 3* ([Example 4, Pattern B](#example-4-materialization-of-window-summaries-in-physical-planning)). The logical candidate is the `Tumbling(1m)` KLL option from Algorithm 2. The diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index eb5cea838..373d4ad3a 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -78,7 +78,7 @@ same pipeline behind the `OptimizationPass` trait. | What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `summary_type/capability.rs` | | Stage 2: the timings and lifetimes offered per logical candidate and their prune rules, the per-group offers (ingestion time, across evaluations), the physical edges, and the alternatives no plan can use | `asap-stage2-physical-optimization`: `candidate.rs`, `group.rs`, `edge.rs`, `pruning.rs`; the admissible-edge indexes in `asap-types`, `stages/physical/memo_candidates/mod.rs` | | Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state/mod.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | -| Stage 3: the selectable edges (`selectable.rs`), each candidate's cost $w_v$ and each edge's $c_{u,c,i}$ (`cost.rs`), retention (`retention.rs`), the baseline MILP (`milp/`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | +| Stage 3: the execution contexts and the executions and edges a plan may select (`selectable.rs`), each execution's and each edge's cost and each retaining candidate's $\pi_v$ (`cost.rs`), how far back each edge reads its producer (`retention.rs`), the selection MILP over execution contexts with $h_v$ and (R) (`milp/`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | | The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | | The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | From ca62eb7d2729500526b5c6aefbd6ac7760a1b72b Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:52:18 +0000 Subject: [PATCH 02/23] feat(stage3): a query reads the groups Pass 2 lists as exact-only for it through exact candidates only: per-query reads a_{q,e} and constraint (X) (#1207 PR 7) --- crates/planner/tests/exact_only_reads.rs | 112 +++++++++++++ crates/planner/tests/main.rs | 1 + crates/stage3-plan-selection/src/lib.rs | 6 +- .../src/milp/exact_only.rs | 158 ++++++++++++++++++ crates/stage3-plan-selection/src/milp/mod.rs | 4 +- .../src/milp/model/mod.rs | 10 +- .../src/milp/model/tests.rs | 79 ++++++++- .../src/milp/upper_bound.rs | 8 +- .../src/stages/logical/memo_candidates.rs | 4 +- docs/design_docs/concepts/accuracy-models.md | 12 +- 10 files changed, 378 insertions(+), 16 deletions(-) create mode 100644 crates/planner/tests/exact_only_reads.rs create mode 100644 crates/stage3-plan-selection/src/milp/exact_only.rs diff --git a/crates/planner/tests/exact_only_reads.rs b/crates/planner/tests/exact_only_reads.rs new file mode 100644 index 000000000..a022ea53d --- /dev/null +++ b/crates/planner/tests/exact_only_reads.rs @@ -0,0 +1,112 @@ +//! A query reads the groups Pass 2 lists as exact-only for it +//! (`LogicalASAPMemoCandidates::exact_only_groups_of_query`) only through +//! exact candidates, while another query of the same workload may still +//! read a sketch of those groups: Stage 3's constraint (X), end to end. + +use std::collections::BTreeSet; + +use asap_planner::{plan_installable, FrontendInput, PlanningModels, UserInput}; +use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::workload::PlanningWorkload; + +/// PromQL `queries`, each with its requirements' JSON (`""`: none), every +/// 60 s at phase 0, over 1,000 gauge series of `m` sampled every second. +fn workload(queries: &[(&str, &str)]) -> PlanningWorkload { + let entries: Vec = queries + .iter() + .map(|(query, requirements)| { + format!( + r#"{{"query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 60000, "evaluation_phase": 0}}}}, + "requirements": {{{requirements}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}"# + ) + }) + .collect(); + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, + "sources": {{"m": {{"series_count": 1000, "metric_type": "gauge", + "sample_interval": 1000}}}}}}}}"#, + entries.join(",") + )) + .unwrap() +} + +/// The plan of `queries`. +async fn plan(queries: &[(&str, &str)]) -> PhysicalASAPDAG { + let workload = workload(queries); + plan_installable(UserInput::new( + &workload, + FrontendInput::Promql, + PlanningModels::builtin(), + )) + .await + .expect("the workload plans") + .0 + .dag +} + +/// The payloads of the nodes at or below query `query`'s root in `dag`. +fn payloads_read_by(dag: &PhysicalASAPDAG, query: usize) -> Vec { + let mut below = BTreeSet::new(); + let mut pending: Vec = vec![dag.roots[query]]; + while let Some(id) = pending.pop() { + if below.insert(id) { + pending.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == id) + .map(|edge| edge.producer), + ); + } + } + dag.nodes + .iter() + .filter(|node| below.contains(&node.id)) + .map(|node| format!("{:?}", node.payload)) + .collect() +} + +/// The same p90 asked at rank error 0.01 (Q0) and at relative error 0.05 +/// (Q1): KLL's rank error does not meet Q1's relative target, so Pass 2 +/// lists the KLL readout and every group below it as exact-only for Q1. +/// Q1 reads no KLL sketch, even where Q0's plan has one to share. +#[tokio::test] +async fn a_relative_query_reads_no_rank_sketch_of_a_query_sharing_its_groups() { + let dag = plan(&[ + ( + "quantile_over_time(0.9, m[5m])", + r#""accuracy": {"max_error": 0.01, "accuracy_metric": "rank"}"#, + ), + ( + "quantile_over_time(0.9, m[5m])", + r#""accuracy": {"max_error": 0.05, "accuracy_metric": "relative"}"#, + ), + ]) + .await; + let read = payloads_read_by(&dag, 1); + assert!( + !read.iter().any(|payload| payload.contains("Sketch(Kll)")), + "the relative query reads a KLL sketch: {read:?}" + ); +} + +/// An exact p90 (no accuracy target) and the same p90 at rank error 0.01: +/// the exact query reads no sketch, whichever plan the other query gets. +#[tokio::test] +async fn an_exact_query_sharing_groups_with_an_approximate_one_reads_no_sketch() { + let dag = plan(&[ + ("quantile_over_time(0.9, m[5m])", ""), + ( + "quantile_over_time(0.9, m[5m])", + r#""accuracy": {"max_error": 0.01, "accuracy_metric": "rank"}"#, + ), + ]) + .await; + let read = payloads_read_by(&dag, 0); + assert!( + !read.iter().any(|payload| payload.contains("Sketch(")), + "the exact query reads a sketch: {read:?}" + ); +} diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index 9d06c8c46..d17ff5aa4 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -6,6 +6,7 @@ mod time_series; mod columnar_batches; mod e2e_plan; +mod exact_only_reads; mod runtime_install; mod sql_table_scans; mod summary_sharing; diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 4d44bc010..fc9c98b01 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -11,8 +11,10 @@ //! $x_{\kappa,v}$ per execution (an `AcrossEvaluations` candidate runs //! once, $x^{\mathrm{mat}}_v$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ and //! the retention $h_v$, under constraints (A)–(F) per context, the -//! one-consumer rule and (R) ((G) and (H) hold because $y$ exists only on -//! admissible edges whose producer delivers on the consumer's context). It +//! one-consumer rule, (R) ((G) and (H) hold because $y$ exists only on +//! admissible edges whose producer delivers on the consumer's context) and +//! (X): a query reads the groups Pass 2 lists as exact-only for it through +//! exact candidates only, $a_{q,e}$ marking what its selected path reads. It //! builds the plan (`PhysicalASAPDAG`) from the selected executions and //! edges as its last step ([`plan`], each node retaining its candidate's //! $h_v$, the needs from [`retention`]). Accuracy at each query root, diff --git a/crates/stage3-plan-selection/src/milp/exact_only.rs b/crates/stage3-plan-selection/src/milp/exact_only.rs new file mode 100644 index 000000000..8121db80b --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/exact_only.rs @@ -0,0 +1,158 @@ +//! Exact-only reads per query +//! (`docs/design_docs/concepts/accuracy-models.md`, "the groups it may read +//! only exactly"): a query reads the groups Pass 2 lists as exact-only for +//! it (`LogicalASAPMemoCandidates::exact_only_groups_of_query`) through +//! exact candidates only, in its own selection context, while another query +//! may still read a sketch of those groups through its own path. +//! +//! $a_{q,e} \in [0, 1]$ marks the executions $e$ query $q$'s selected path +//! reads, from its output down; constraint (X) keeps it at 0 on every +//! execution $q$ must not read. + +use std::collections::{BTreeMap, BTreeSet}; + +use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; + +use super::model::{QueryId, SelectionInputs}; +use crate::selectable::Execution; + +/// Whether query `query` must not read `execution`: its candidate is not +/// exact (it has sketch parameters θ) and its group is one the query may +/// read only exactly. A readout's own candidate has no θ, but the sketch +/// states it reads are in the same list, as every group below a listed one +/// is. +/// +/// Example: `quantile_over_time(0.9, m[5m])` at a relative target lists +/// the quantile and every group below it; its KLL state builds (θ) are +/// executions it must not read, the exact quantile one it may. +pub(crate) fn must_not_read( + inputs: &SelectionInputs<'_>, + query: QueryId, + execution: Execution, +) -> bool { + let logical = &inputs.physical.logical_candidates; + let id = inputs.physical.physical_candidates[execution.candidate.0].logical_candidate_id; + logical + .exact_only_groups_of_query + .get(query.0) + .is_some_and(|groups| groups.contains(&id.group)) + && logical + .memo + .candidate(id) + .logical_properties + .parameters() + .is_some() +} + +/// $a_{q,e}$, added to `problem`, for each query $q$ and each execution $e$ +/// it may reach from its output executions over selectable edges (`y`) +/// from which an execution it must not read ([`must_not_read`]) is +/// reachable, less those themselves. None for a query with no exact-only +/// group. +pub(crate) fn exact_only_variables( + inputs: &SelectionInputs<'_>, + y: &BTreeMap, + problem: &mut ProblemVariables, +) -> BTreeMap<(QueryId, Execution), Variable> { + let contexts = inputs.contexts; + let count = contexts.executions().len(); + // The selectable edges, as producers per consumer and back. + let mut producers = vec![Vec::new(); count]; + let mut consumers = vec![Vec::new(); count]; + for &position in y.keys() { + let (consumer, _, producer) = contexts.edges()[position]; + let (c, u) = (contexts.index_of(consumer), contexts.index_of(producer)); + producers[c].push(u); + consumers[u].push(c); + } + let execution = |e: usize| contexts.executions()[e]; + let queries: BTreeSet = inputs.query_outputs.iter().map(|(q, _)| *q).collect(); + let mut tracked = BTreeMap::new(); + for query in queries { + let mut reached = vec![false; count]; + let mut pending: Vec = inputs + .query_outputs + .iter() + .filter(|(q, _)| *q == query) + .flat_map(|(_, executions)| executions.iter().map(|&e| contexts.index_of(e))) + .collect(); + while let Some(e) = pending.pop() { + if !std::mem::replace(&mut reached[e], true) { + pending.extend(&producers[e]); + } + } + let mut reaches_forbidden = vec![false; count]; + let mut pending: Vec = (0..count) + .filter(|&e| reached[e] && must_not_read(inputs, query, execution(e))) + .collect(); + while let Some(e) = pending.pop() { + if reached[e] && !std::mem::replace(&mut reaches_forbidden[e], true) { + pending.extend(&consumers[e]); + } + } + for e in (0..count) + .filter(|&e| reaches_forbidden[e] && !must_not_read(inputs, query, execution(e))) + { + tracked.insert((query, execution(e)), problem.add(variable().min(0).max(1))); + } + } + tracked +} + +/// (X) Query $q$ reads the groups it may read only exactly through exact +/// candidates: $a_{q,e} \ge z_{q,e}$ for an output execution, +/// $a_{q,p} \ge y_{\kappa,p,c,i} + a_{q,c} - 1$ along a selectable edge, so +/// $a_{q,e} = 1$ on every execution $q$'s selected path reads; an execution +/// $q$ must not read ([`must_not_read`]) has $a_{q,e} = 0$: $z_{q,e} = 0$ +/// and $y_{\kappa,p,c,i} + a_{q,c} \le 1$. Other queries may still read it, +/// through their own paths. $a$ exists only where [`exact_only_variables`] +/// tracks it, and is not in the objective. +pub(crate) fn exact_only_constraints( + inputs: &SelectionInputs<'_>, + a: &BTreeMap<(QueryId, Execution), Variable>, + y: &BTreeMap, + z: &BTreeMap<(QueryId, Execution), Variable>, +) -> Vec { + let mut constraints = Vec::new(); + for (&(query, execution), &z) in z { + if must_not_read(inputs, query, execution) { + constraints.push(constraint!(z <= 0)); + } else if let Some(&a) = a.get(&(query, execution)) { + constraints.push(constraint!(a >= z)); + } + } + // The tracked queries, per execution. + let mut readers: BTreeMap> = BTreeMap::new(); + for (&(query, execution), &a) in a { + readers.entry(execution).or_default().push((query, a)); + } + for (&position, &y) in y { + let (consumer, _, producer) = inputs.contexts.edges()[position]; + for &(query, a_consumer) in readers.get(&consumer).into_iter().flatten() { + if must_not_read(inputs, query, producer) { + constraints.push(constraint!(y + a_consumer <= 1)); + } else if let Some(&a_producer) = a.get(&(query, producer)) { + constraints.push(constraint!(a_producer >= y + a_consumer - 1)); + } + } + } + constraints +} + +/// Per execution, by position in `ExecutionContexts::executions`, whether +/// some query must not read it ([`must_not_read`]): the greedy bound +/// ([`super::upper_bound`]) leaves them out, so the plan it finds meets (X) +/// whichever query reads what. +pub(crate) fn not_read_by_some_query(inputs: &SelectionInputs<'_>) -> Vec { + let queries: BTreeSet = inputs.query_outputs.iter().map(|(q, _)| *q).collect(); + inputs + .contexts + .executions() + .iter() + .map(|&execution| { + queries + .iter() + .any(|&query| must_not_read(inputs, query, execution)) + }) + .collect() +} diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index 2e893922d..f0b474911 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -2,10 +2,12 @@ //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! sections 3–5): its variables over execution contexts and one function //! per constraint family ([`model`]), the retention amount $h_v$ and -//! constraint (R) ([`execution_contexts`]), the graph algorithm it needs +//! constraint (R) ([`execution_contexts`]), each query's exact-only reads, +//! constraint (X) ([`exact_only`]), the graph algorithm it needs //! ([`graph`]), the bound it scales by ([`upper_bound`]), and the accuracy //! extension ([`accuracy`]). mod accuracy; +mod exact_only; mod execution_contexts; mod graph; pub(crate) mod model; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 188433c32..0a5826d75 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -6,6 +6,7 @@ //! |---|---|---| //! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_constraints`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | //! | retention (section 5) | $h_v$ | (R) [`super::execution_contexts::retention_constraints`] | +//! | exact-only reads per query | $a_{q,e}$ | (X) [`super::exact_only::exact_only_constraints`] | //! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ (`AccuracyVariables`) | (K) `accuracy_constraints` | //! //! [`solve_selection`] solves it without (K), whose variables and @@ -29,6 +30,7 @@ use asap_types::stages::{ }; use asap_types::workload::QueryWorkloadEntry; +use super::exact_only::{exact_only_constraints, exact_only_variables}; use super::execution_contexts::{least_retention_ms, retention_constraints, retention_variables}; use super::graph::Components; use super::upper_bound::dearer_than_a_feasible_plan; @@ -75,6 +77,9 @@ pub(crate) struct SelectionVariables { /// cost per second of keeping it, at least each selected scan's /// ([`raw_data_disk_constraints`]). raw_data_on_disk: BTreeMap, + /// $a_{q,e}$: query $q$'s selected path reads execution $e$, where + /// (X) needs it ([`exact_only_variables`]). + read_by_query: BTreeMap<(QueryId, Execution), Variable>, } /// $x$, $y$, $z$, $t$, $h$ and $r$ over `inputs`, added to `problem`: $x$ @@ -130,6 +135,7 @@ fn selection_variables( .entry(source.clone()) .or_insert_with(|| problem.add(variable().min(0))); } + let read_by_query = exact_only_variables(inputs, &y, problem); SelectionVariables { x, y, @@ -137,6 +143,7 @@ fn selection_variables( t, h, raw_data_on_disk, + read_by_query, } } @@ -343,7 +350,7 @@ pub(crate) enum SelectionMilpError { /// \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum \rho_\kappa c_{\kappa,u,c,i} /// y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s r_s$ ($\pi_v h_v$ of a /// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], -/// [`raw_data_disk_constraints`] and (R), with HiGHS at gap 0 on one +/// [`raw_data_disk_constraints`], (R) and (X), with HiGHS at gap 0 on one /// thread. The executions dearer than a feasible plan are left out first /// ([`dearer_than_a_feasible_plan`]), then the coefficients are scaled so /// the largest left is 1,000 (HiGHS's tolerances are absolute). @@ -363,6 +370,7 @@ pub(crate) fn solve_selection( use_constraints(inputs, &variables), raw_data_disk_constraints(inputs, &variables), retention_constraints(inputs, &variables.h, &variables.y, &variables.z), + exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), ] .into_iter() .flatten() diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 9f5987bff..25f66fb56 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -2,7 +2,7 @@ //! costs: which executions and edges its optimum selects, its retention, //! and the plan built from it. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; @@ -15,9 +15,9 @@ use asap_types::stages::{ PhysicalEdgeId, }; use asap_types::stages::{Lifetime, PhysicalProperties, StorageMedium}; -use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; +use asap_types::summary_type::{ExactKind, ExactParams, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_builders::{ - group, own_candidate, query, scan_group, try_candidate, + group, own_candidate, quantile_over_time_readouts, query, scan_group, try_candidate, }; use asap_types::test_support::memo_fixtures; use asap_types::workload::DataWorkload; @@ -492,3 +492,76 @@ fn a_candidate_dearer_than_a_feasible_plan_does_not_hide_a_cheaper_plan() { let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); assert_eq!(candidates(&solved.unwrap()), vec![3, 4, 5, 6, 7, 8]); } + +/// `quantile_over_time(0.9, m[5m])` read by Q0 every 10 s and Q1 every +/// 30 s, two contexts: its KLL readout (cost 1 per execution, through the +/// state group) is cheaper than the exact quantile (cost 100). Q1 may read +/// its exact-only groups only exactly (a relative target, which the KLL +/// readout's rank error does not meet). With Pass 2's list (the quantile +/// and every group below it) Q1's output is the exact quantile, constraint +/// (X) at the output; with only the state group listed it is too, as the +/// readout would read the state for Q1 (X along the edge); with no list Q1 +/// reads the readout. Q0, with no list, reads the readout throughout. +#[test] +fn a_query_reads_its_exact_only_groups_through_exact_candidates_only() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let window = range(&mut memo, 300, m); + let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); + let state = quantile.by_algorithm[0].state; + query(&mut memo, quantile.result, None); + query(&mut memo, quantile.result, None); + // Every candidate, one θ of the state's. + let logical_candidates: Vec<(CandidateForMemoGroupId, Lifetime)> = (0..memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .flat_map(|group| { + let count = match group == state { + true => 1, + false => memo.groups()[group.0 as usize].candidates.len(), + }; + (0..count).map(move |index_in_group| { + ( + CandidateForMemoGroupId { + group, + index_in_group, + }, + ONE_EVALUATION, + ) + }) + }) + .collect(); + let exact_quantile = logical_candidates + .iter() + .position(|&(candidate, _)| candidate == own_candidate(quantile.result)) + .unwrap(); + let costs: Vec = (0..logical_candidates.len()) + .map(|v| if v == exact_quantile { 100.0 } else { 1.0 }) + .collect(); + let mut physical = physical(memo, &logical_candidates); + // Every group: the quantile and every group below it. + let below_quantile: BTreeSet = + (0..physical.logical_candidates.memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .collect(); + for (q1_exact_only, q1_reads_exactly) in [ + (below_quantile, true), + (BTreeSet::from([state]), true), + (BTreeSet::new(), false), + ] { + physical.logical_candidates.exact_only_groups_of_query = + vec![BTreeSet::new(), q1_exact_only]; + let (solved, _) = solve(&physical, &[every(10_000), every(30_000)], &costs, &[], &[]); + let outputs: Vec = solved + .unwrap() + .outputs + .iter() + .map(|execution| execution.candidate.0) + .collect(); + assert_ne!(outputs[0], exact_quantile); + assert_eq!( + outputs[1] == exact_quantile, + q1_reads_exactly, + "{outputs:?}" + ); + } +} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index a4bf445c4..ac29dbddb 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use asap_types::ir::properties::ExecutionTiming; use asap_types::stages::Lifetime; +use super::exact_only::not_read_by_some_query; use super::model::{selection_cost_per_second, QueryId, SelectionInputs}; use crate::selectable::Execution; @@ -80,8 +81,9 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec) -> Option { let physical = inputs.physical; let contexts = inputs.contexts; @@ -113,7 +115,7 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { let properties = physical.physical_candidates[v.0].physical_properties; properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime }; - let mut excluded = vec![false; count]; + let mut excluded = not_read_by_some_query(inputs); for _ in 0..REPAIRS { let (tree_cost, chosen) = cheapest_trees(inputs, &into, &inputs_of, &excluded); // Each query output's cheapest execution, then the trees below. diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 16125313f..bce6930d6 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -36,8 +36,8 @@ pub struct LogicalASAPMemoCandidates { /// unknown) also lists each readout group another query gave a level, /// with the groups below it. A group is listed for a query, not /// pruned: other queries may still read its approximate candidates, so - /// Stage 3 must enforce it per query, in each query's selection - /// context; Pass 2 only lists it. Empty for a query without an accuracy + /// Stage 3 enforces it per query, in each query's selection context + /// (its MILP's constraint (X)); Pass 2 only lists it. Empty for a query without an accuracy /// target. pub exact_only_groups_of_query: Vec>, } diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index f59c6a3f0..633812520 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -301,10 +301,14 @@ does not allow. With Q0 = `p99 / p50` and Q1 = `p99 / p50 > 1` sharing the ratio, Q0 may read the DDSketch readouts while Q1 reads the ratio and every group below it only exactly; with `quantile_over_time(0.9, m[5m])` asked once at a rank target and once at a relative one, each reads the other's readout -groups only exactly. Stage 3 must enforce the list per query; Pass 2 only -computes it. A scalar root's operands are roots: `scalar(v)` and a scalar -subquery keep the target's metrics, and any other scalar expression reads its -operands only exactly. +groups only exactly. Pass 2 only computes the list; Stage 3 enforces it per +query, in the query's own selection context (the selection MILP's +constraint (X)): no execution on the query's selected path, from its output +down, in a group on its list is a candidate with sketch parameters, while +another query may still read that candidate through its own path. A scalar +root's operands are roots: `scalar(v)` and a scalar subquery keep the +target's metrics, and any other scalar expression reads its operands only +exactly. The list is per group, so it is conservative where one group holds candidates a query may and may not read: `avg by (job)(count_over_time(m[1m]))` From f22af5e65cfcca79421518a4e34ec2583ac744fd Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:29:05 +0000 Subject: [PATCH 03/23] fix(stage3): one run rate shared with plan_cost, request contexts per anchor, retention and scale over kept executions, per-query bound (#1207 PR 7) - asap-cost: context_executions_per_second is the one run-rate function Stage 3 and plan_cost both call; plan_cost walks through OneEvaluation nodes, so an `@ end()` / `@ start()` subtree runs once per request in both. `@ start()` requests count once per distinct start time. - Inputs of `@ end()` and `@ start()` shifts run in contexts of their own, apart from each other and from plain evaluations. - h_v and its (R) rows exist only for a candidate with a kept execution; the objective scale is over kept coefficients only. - The greedy bound leaves out of each query's trees only what that query must not read. - A query-time Scan that is not materialized has its read cost on its edges; a materialized one keeps a per-execution read. - SelectableExecutions (renamed) holds executions, edges by execution position, edges per input and the answering executions; milp/retention.rs holds h_v and (R); QueryId moves to selectable.rs. - pi_v is stored per second retained; each edge's need is computed once and stored on ExecutionCosts; the request overhead is added in select_plan; forbidden answering executions are filtered before the MILP instead of (X) rows z <= 0. - PhysicalProperties::is_across_evaluations, PhysicalCandidate::own_schedule, AtModifier derives Hash and Ord, Consumer::QueryRequest carries the anchor. --- crates/cost/src/plan/execution_demand/mod.rs | 251 ++++++++------ .../cost/src/plan/execution_demand/tests.rs | 88 +++++ .../tests/stage3_objective.rs | 124 +++++++ crates/stage3-plan-selection/src/cost.rs | 308 ++++++++++------- .../stage3-plan-selection/src/cost/tests.rs | 30 +- crates/stage3-plan-selection/src/lib.rs | 43 +-- .../src/milp/exact_only.rs | 115 +++---- .../src/milp/execution_contexts.rs | 108 ------ .../stage3-plan-selection/src/milp/graph.rs | 12 +- crates/stage3-plan-selection/src/milp/mod.rs | 6 +- .../src/milp/model/mod.rs | 281 ++++++++-------- .../src/milp/model/tests.rs | 189 ++++++++--- .../src/milp/retention.rs | 117 +++++++ .../src/milp/upper_bound.rs | 314 +++++++++--------- crates/stage3-plan-selection/src/plan.rs | 10 +- crates/stage3-plan-selection/src/retention.rs | 87 ++--- .../stage3-plan-selection/src/selectable.rs | 252 ++++++++------ crates/types/src/ir/operator/arguments.rs | 2 +- crates/types/src/stages/physical/candidate.rs | 38 ++- .../src/stages/physical/properties/mod.rs | 6 + 20 files changed, 1404 insertions(+), 977 deletions(-) delete mode 100644 crates/stage3-plan-selection/src/milp/execution_contexts.rs create mode 100644 crates/stage3-plan-selection/src/milp/retention.rs diff --git a/crates/cost/src/plan/execution_demand/mod.rs b/crates/cost/src/plan/execution_demand/mod.rs index dc022f87f..c07259f98 100644 --- a/crates/cost/src/plan/execution_demand/mod.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -15,6 +15,7 @@ use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; use asap_types::stages::Lifetime; use asap_types::workload::{ greatest_common_divisor, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand, + TimestampMs, }; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -285,30 +286,31 @@ pub fn executions_per_second<'e>( ) } -/// What computes a not-materialized node once per its own execution -/// ([`consumers`]). +/// What computes a node that is not materialized across evaluations once +/// per its own execution ([`consumers`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Consumer { /// Query `.0` (an operator or scalar root), at its own rate. Query(usize), - /// Query `query` below a `TimeShift` anchored with `@ end()` or (when - /// `at_start`) `@ start()`: once per request, on the grid of its - /// request end (start) times ([`request_time_grid_ms`]). - QueryRequest { query: usize, at_start: bool }, - /// A materialized node, at that node's rate. + /// Query `query` below a `TimeShift` anchored with `at` (`@ end()` or + /// `@ start()`): once per request ([`context_executions_per_second`]). + QueryRequest { query: usize, at: AtModifier }, + /// A node materialized across evaluations, at that node's rate. Materialized(PhysicalASAPNodeId), } -/// The consumers of each not-materialized node of `dag` that any consumer -/// reaches ([`computations`]): the queries and the materialized nodes whose -/// walk down their inputs reaches it before meeting a materialized node, -/// each once; the queries first, in order, then the materialized nodes in -/// `dag`'s node order. +/// The consumers of each node of `dag` not materialized across evaluations +/// that any consumer reaches ([`computations`]): the queries and the nodes +/// materialized across evaluations whose walk down their inputs reaches it +/// before meeting such a node, each once; the queries first, in order, +/// then the materialized nodes in `dag`'s node order. A `OneEvaluation` +/// node is computed within its consumers' context, so a walk goes through +/// it. pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap> { let materialized_nodes: Vec = dag .nodes .iter() - .filter(|n| n.lifetime != Lifetime::None) + .filter(|n| matches!(n.lifetime, Lifetime::AcrossEvaluations(_))) .map(|n| n.id) .collect(); let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); @@ -318,21 +320,18 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap match shift.at { - Some(AtModifier::End) => Some(false), - Some(AtModifier::Start) => Some(true), - _ => None, - }, + // The anchor of each `TimeShift` anchored with `@ end()` or `@ start()`. + let request_anchor: HashMap = dag + .nodes + .iter() + .filter_map(|node| match &node.payload { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { + Some(at @ (AtModifier::End | AtModifier::Start)) => Some((node.id, at)), _ => None, - }) - }; + }, + _ => None, + }) + .collect(); let walks = query_starts(dag) .into_iter() .enumerate() @@ -354,10 +353,8 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap { - Consumer::QueryRequest { query, at_start } - } + let below = match (consumer, request_anchor.get(&id)) { + (Consumer::Query(query), Some(&at)) => Consumer::QueryRequest { query, at }, (consumer, _) => consumer, }; stack.extend(inputs(id).map(|input| (input, below))); @@ -366,59 +363,121 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap u64 { + entry + .recurrence + .evaluation_grid_ms(entry.range_evaluation) + .unwrap_or(1) +} + +/// The grid, in ms, of the times `entry`'s requests anchor `@ end()` or +/// `@ start()` (`at`) at: its request end times, shifted back by its +/// range-evaluation span for `@ start()`; 1 ms when not known. /// /// Example: a query every 60 s over the last hour: `@ end()` 60,000 ms, /// `@ start()` gcd(60,000, 3,600,000) = 60,000 ms. -pub fn request_time_grid_ms( - recurrence: &QueryRecurrence, - range_evaluation: Option, - at: AtModifier, -) -> u64 { - let end_ms = recurrence.evaluation_grid_ms(None).unwrap_or(1); - match (at, range_evaluation) { +pub fn request_time_grid_ms(entry: &QueryWorkloadEntry, at: AtModifier) -> u64 { + let end_ms = entry.recurrence.evaluation_grid_ms(None).unwrap_or(1); + match (at, entry.range_evaluation) { (AtModifier::Start, Some(range)) => greatest_common_divisor(end_ms, range.span_ms), _ => end_ms, } } +/// How many times per second a node runs in one execution context of the +/// queries `queries` (a node not materialized across evaluations, run once +/// per execution of the context; Stage 3 and [`computations`] both cost it +/// so): with no `anchor`, at every evaluation of theirs, a range query's +/// steps each one ([`evaluation_rate`]); below a `TimeShift` anchored with +/// `anchor` (`@ end()`, `@ start()`), once per distinct anchor time of +/// their requests: a request's end, or its end less its range-evaluation +/// span. +/// +/// Example: two range queries every 60 s at phase 0 with steps of 30 s, +/// spans 1 h and 90 s, below `@ start()`: their requests start at +/// `k · 60 s` and `k · 60 s + 30 s`, 2/60 per second; below `@ end()` +/// both end at `k · 60 s`, 1/60 per second. +pub fn context_executions_per_second<'e>( + anchor: Option, + queries: impl IntoIterator, + non_repeating_query_horizon_seconds: f64, +) -> Result { + let per_evaluation: Vec<(QueryRecurrence, Option)> = queries + .into_iter() + .map(|entry| match anchor { + None => (entry.recurrence.clone(), entry.range_evaluation), + Some(at) => (anchored_request_recurrence(entry, at), None), + }) + .collect(); + evaluation_rate( + per_evaluation + .iter() + .map(|(recurrence, range)| (recurrence, *range)), + non_repeating_query_horizon_seconds, + ) +} + +/// The recurrence of the times `entry`'s requests anchor `at` at: its own, +/// with a fixed interval's phase moved back by the range-evaluation span +/// for `@ start()`. +fn anchored_request_recurrence(entry: &QueryWorkloadEntry, at: AtModifier) -> QueryRecurrence { + match (&entry.recurrence, at, entry.range_evaluation) { + ( + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval, + evaluation_phase, + }), + AtModifier::Start, + Some(range), + ) if interval.0 > 0 => { + let interval_ms = u64::from(interval.0); + let phase_ms = (evaluation_phase.0 % interval_ms + interval_ms + - range.span_ms % interval_ms) + % interval_ms; + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: *interval, + evaluation_phase: TimestampMs(phase_ms), + }) + } + (recurrence, _, _) => recurrence.clone(), + } +} + /// How often each node of `dag` is computed: `(multiplier, per_second)` -/// (module doc, "Cost model"). A materialized node is computed once for all -/// its queries: once per second at ingestion time (materialized at ingestion time), at -/// the combined rate of the queries reaching it (`OneEvaluation`, -/// [`evaluation_rate`]), or once per request of those queries for a chain -/// materialized across evaluations, whose request builds only its newest tumbling window -/// however many range steps it evaluates ([`executions_per_second`]). -/// `entries[q]` is query `q`'s workload entry (its recurrence and range -/// evaluation). A not-materialized node is computed once per execution -/// context of the consumers that reach it ([`consumers`]; the execution -/// contexts of the concepts doc, section 5). A consumer is a query -/// (operator or scalar root) or a materialized node; a walk stops at the -/// first materialized node it meets, and a node read several times within -/// one consumer's walk counts once for it. The queries on one evaluation -/// grid (`QueryRecurrence::evaluation_grid_ms` of their range evaluation; -/// 1 ms when not known) are one context, which computes the node at their -/// combined rate ([`evaluation_rate`]); below a `TimeShift` anchored with -/// `@ end()` or `@ start()` a query computes the node once per request, on -/// the grid of its request end (start) times ([`request_time_grid_ms`]); -/// a materialized node is a context of its own, at its rate. Its -/// `multiplier` is that number of contexts. A node computed at -/// ingestion time, materialized or not, computes each arriving row once: -/// once per second, `(1, 1.0)`, however many ingestion-time views of -/// however many refresh windows read it. +/// (module doc, "Cost model"). A node materialized across evaluations is +/// computed once for all its queries: once per second at ingestion time, +/// or once per request of the queries reaching it at query time, a chain +/// building only its newest tumbling window however many range steps it +/// evaluates ([`executions_per_second`]). `entries[q]` is query `q`'s +/// workload entry (its recurrence and range evaluation). Any other node +/// (ℓ None or `OneEvaluation`) is computed once per execution context of +/// the consumers that reach it ([`consumers`]; the execution contexts of +/// the concepts doc, section 5). A consumer is a query (operator or scalar +/// root) or a node materialized across evaluations; a walk stops at the +/// first such node it meets, and a node read several times within one +/// consumer's walk counts once for it. The queries on one evaluation grid +/// ([`evaluation_grid_ms`]) are one context, and the requests of the +/// queries on one grid below a `TimeShift` anchored with `@ end()` (or +/// `@ start()`) another, each at [`context_executions_per_second`]; a +/// materialized node is a context of its own, at its rate. Its +/// `multiplier` is that number of contexts. A node computed at ingestion +/// time, materialized or not, computes each arriving row once: once per +/// second, `(1, 1.0)`, however many ingestion-time views of however many +/// refresh windows read it. pub fn computations( dag: &PhysicalASAPDAG, entries: &[QueryWorkloadEntry], non_repeating_query_horizon_seconds: f64, ) -> Result, AnalyticalCostError> { - // Each materialized node's timing and lifetime. + // Each node materialized across evaluations: its timing and lifetime. let materialized_nodes: HashMap = dag .nodes .iter() - .filter(|node| node.lifetime != Lifetime::None) + .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| (node.id, (node.output_state.timing, node.lifetime))) .collect(); let inputs = |id: PhysicalASAPNodeId| { @@ -456,57 +515,33 @@ pub fn computations( )?; rate_of_materialized.insert(id, rate); } - // Each query's evaluation grid: the queries on one grid are one - // execution context, evaluated together. - let grid_ms = |query: usize| { - entries - .get(query) - .and_then(|entry| entry.recurrence.evaluation_grid_ms(entry.range_evaluation)) - .unwrap_or(1) - }; let mut count: HashMap<_, (u64, f64)> = HashMap::new(); for (id, of_node) in consumers(dag) { let entry = count.entry(id).or_default(); - // The queries reaching it, by evaluation grid: each evaluated at - // its steps, or once per request (below an `@ end()` / `@ start()` - // shift) when it reaches the node only so on that grid. - let mut queries_by_grid: BTreeMap> = BTreeMap::new(); + // The queries reaching it, per context: (evaluation grid, anchor). + let mut queries_by_context: BTreeMap<(u64, Option), BTreeSet> = + BTreeMap::new(); for consumer in of_node { - match consumer { - Consumer::Query(query) => { - queries_by_grid - .entry(grid_ms(query)) - .or_default() - .insert(query, false); - } - Consumer::QueryRequest { query, at_start } => { - let at = match at_start { - true => AtModifier::Start, - false => AtModifier::End, - }; - let grid = entries.get(query).map_or(1, |entry| { - request_time_grid_ms(&entry.recurrence, entry.range_evaluation, at) - }); - queries_by_grid - .entry(grid) - .or_default() - .entry(query) - .or_insert(true); - } + let (query, anchor) = match consumer { + Consumer::Query(query) => (query, None), + Consumer::QueryRequest { query, at } => (query, Some(at)), Consumer::Materialized(node) => { entry.0 += 1; entry.1 += rate_of_materialized[&node]; + continue; } - } + }; + let grid_ms = entries.get(query).map_or(1, evaluation_grid_ms); + queries_by_context + .entry((grid_ms, anchor)) + .or_default() + .insert(query); } - for queries in queries_by_grid.into_values() { + for ((_, anchor), queries) in queries_by_context { entry.0 += 1; - entry.1 += evaluation_rate( - queries.iter().filter_map(|(&query, &per_request)| { - let entry = entries.get(query)?; - let range = (!per_request).then_some(entry.range_evaluation).flatten(); - Some((&entry.recurrence, range)) - }), + entry.1 += context_executions_per_second( + anchor, + queries.iter().filter_map(|&query| entries.get(query)), non_repeating_query_horizon_seconds, )?; } diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs index 9cc26d4c2..27e13d846 100644 --- a/crates/cost/src/plan/execution_demand/tests.rs +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -298,6 +298,94 @@ fn a_range_query_computes_its_at_end_subtree_once_per_request() { } } +/// `m[5m]` below a `TimeShift` anchored with `at`, the shift a +/// `OneEvaluation` result read by every query (`roots`). +fn anchored_dag(at: AtModifier, roots: Vec) -> PhysicalASAPDAG { + use asap_types::ir::operator::TimeShift; + use asap_types::stages::StorageMedium; + let query_time = ExecutionTiming::QueryTime; + let shift = Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(at), + }, + child: 1, + }); + PhysicalASAPDAG::new( + vec![ + node(0, scan(), query_time, Lifetime::None), + node(1, last(300, 0), query_time, Lifetime::None), + node( + 2, + shift, + query_time, + Lifetime::OneEvaluation(StorageMedium::Memory), + ), + ], + roots, + vec![], + ) + .unwrap() +} + +/// Two range queries every 300 s at a 30 s step over 600 s (21 steps per +/// request) share one `OneEvaluation` `@ end()` shift: it runs at every +/// step of their one context, 21/300 per second, and its input once per +/// request, 1/300: a walk goes through a `OneEvaluation` node. +#[test] +fn a_shared_one_evaluation_at_end_shift_runs_its_input_once_per_request() { + let span = RangeEvaluation { + step_ms: 30_000, + span_ms: 600_000, + }; + let recurrence = [fixed(300_000, 0), fixed(300_000, 0)]; + let rates = computations( + &anchored_dag(AtModifier::End, vec![2, 2]), + &entries(&recurrence, Some(span)), + 3600.0, + ) + .unwrap(); + assert_eq!(rates[&2].0, 1); + close(rates[&2].1, 21.0 / 300.0); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 1.0 / 300.0); + } +} + +/// Two range queries every 60 s at a 30 s step, over 1 h and over 90 s, +/// below one `@ start()` shift: one context (one evaluation grid, one +/// anchor), whose requests start at `k · 60 s` and `k · 60 s + 30 s`: the +/// input runs 2/60 per second. +#[test] +fn at_start_inputs_run_once_per_distinct_request_start() { + let with_span = |span_ms| QueryWorkloadEntry { + query: asap_types::workload::Query("q".into()), + requirements: Default::default(), + predictability: Default::default(), + recurrence: fixed(60_000, 0), + range_evaluation: Some(RangeEvaluation { + step_ms: 30_000, + span_ms, + }), + }; + let entries = [with_span(3_600_000), with_span(90_000)]; + let rates = computations( + &anchored_dag(AtModifier::Start, vec![2, 2]), + &entries, + 3600.0, + ) + .unwrap(); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 2.0 / 60.0); + } + close( + context_executions_per_second(Some(AtModifier::End), &entries, 3600.0).unwrap(), + 1.0 / 60.0, + ); +} + // Invalid fixed intervals and estimated rates still produce the evaluator's error. #[test] fn invalid_recurrence_is_rejected() { diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index dccb77b05..3a95d9998 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -317,3 +317,127 @@ fn every_10s(query: &str) -> serde_json::Value { "predictability": {"predictable": {"known_at": null}}, }) } + +/// `query` requested every `interval_ms` at phase 0 as a range query with +/// steps of `step_ms` over `span_ms`, or as an instant query without one. +fn requested(query: &str, interval_ms: u32, range: Option<(u64, u64)>) -> serde_json::Value { + let mut entry = serde_json::json!({ + "query": query, + "demand": {"fixed_interval_at": {"interval": interval_ms, "evaluation_phase": 0}}, + "requirements": {"max_latency_ms": null}, + "predictability": {"predictable": {"known_at": null}}, + }); + if let Some((step_ms, span_ms)) = range { + entry["range_evaluation"] = serde_json::json!({"step_ms": step_ms, "span_ms": span_ms}); + } + entry +} + +/// `entries` over the gauge `m` (1,000 series, one sample per second). +fn over_m(entries: Vec) -> PlanningWorkload { + serde_json::from_value(serde_json::json!({ + "query_workload": {"language": "promql", "repeating_queries": entries}, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 1000.0}}, + "sources": {"m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}}, + }, + })) + .unwrap() +} + +/// Two range queries every 300 s at 30 s steps over 600 s that read one +/// `@ end()` subtree: they share its shift, `OneEvaluation`, and its input +/// runs once per request, in the MILP and in `plan_cost` alike +/// (`plan_cost` charged it at every step: 0.1513/s for an objective of +/// 0.0473/s). +#[test] +fn the_objective_is_the_plan_cost_on_a_shared_at_end_subtree() { + let w = over_m(vec![ + requested( + "sum_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + requested( + "max_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + ]); + assert_equal( + "shared @ end() subtree", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} + +/// Two range queries every 60 s at 30 s steps, over 1 h and over 90 s, +/// below `@ start()`: one context, whose requests start at two times each +/// minute, `k · 60 s` and `k · 60 s + 30 s`; Stage 3 and `plan_cost` count +/// both. +#[test] +fn the_objective_is_the_plan_cost_on_at_start_subtrees_of_two_spans() { + let w = over_m(vec![ + requested( + "max_over_time(m[5m] @ start())", + 60_000, + Some((30_000, 3_600_000)), + ), + requested( + "max_over_time(m[5m] @ start())", + 60_000, + Some((30_000, 90_000)), + ), + ]); + assert_equal( + "@ start() subtrees of two spans", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} + +/// An instant `max_over_time(m[5m])` every 300 s and the range query +/// `max_over_time(m[5m] @ start())` every 300 s at 30 s steps over 600 s: +/// the range query reads `m[5m]` at its requests' start, 600 s before the +/// instant query's evaluation, so no result computed for one evaluation is +/// read by both (a `OneEvaluation` `m[5m]` served both). +#[test] +fn an_at_start_subtree_shares_no_evaluation_result_with_an_instant_query() { + let w = over_m(vec![ + requested("max_over_time(m[5m])", 300_000, None), + requested( + "max_over_time(m[5m] @ start())", + 300_000, + Some((30_000, 600_000)), + ), + ]); + let (plan, objective_and_plan_cost) = + selected_plan_objective_and_plan_cost(&w, &CostInputs::default(), true); + assert_equal("@ start() beside an instant query", objective_and_plan_cost); + let producers = |id| { + plan.edges + .iter() + .filter(move |edge| edge.consumer == id) + .map(|edge| edge.producer) + }; + // The nodes each query's walk reaches below its root, stopping at + // results kept across evaluations or computed at ingestion time. + let reached = |root: usize| { + let mut reached = std::collections::BTreeSet::new(); + let mut pending = vec![root]; + while let Some(id) = pending.pop() { + let node = &plan.nodes[id]; + if matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) + || node.output_state.timing == ExecutionTiming::IngestionTime + || !reached.insert(id) + { + continue; + } + pending.extend(producers(id)); + } + reached + }; + let (instant, at_start) = (reached(plan.roots[0]), reached(plan.roots[1])); + assert!( + instant.is_disjoint(&at_start), + "{instant:?} and {at_start:?} share a node: {plan:#?}" + ); +} diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 90f9bd3d5..753c2172a 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -10,7 +10,8 @@ //! \rho_\kappa c_{\kappa,u,c,i} y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s //! r_s,$$ //! -//! plus the request overhead every plan pays. Each candidate is costed as +//! less the request overhead every plan pays (`crate::select_plan` adds +//! it). Each candidate is costed as //! one plan node ([`candidate_node`], //! [`asap_cost::node_resource_usage_and_cost`]) over a baseline of each //! input: the most any producer it may read (on its admissible edges) @@ -28,9 +29,12 @@ //! producers with it), so the baseline over its admissible producers is //! that θ's. //! -//! An execution of a query-time `Scan` reads, once per run, the most any -//! consumer it may have in its context reads back in one run -//! ([`edge_scan_read_ms`]), a constant of the execution: exact when that +//! A query-time `Scan` that is not materialized has one consumer per +//! context, and reads in each run what that consumer reads back in one +//! run ([`edge_scan_read_ms`]): that read is on the edge to the consumer, +//! a scan's read cost on its edges. A materialized one (`OneEvaluation`, +//! shared within a context) reads the most any consumer it may have in +//! the context reads, a constant of the execution: exact when that //! consumer is selected. //! //! A candidate that retains time-indexed rows @@ -47,12 +51,14 @@ use std::collections::HashMap; -use asap_cost::plan::execution_demand::{evaluation_rate, executions_per_second as rate_of}; +use asap_cost::plan::execution_demand::{ + context_executions_per_second, executions_per_second as rate_of_materialized, +}; use asap_cost::{ maintained_window_ms, node_resource_usage_and_cost, raw_data_disk_cost_per_second, CostInputs, NodeResourceUsageAndCost, ProducerOutput, }; -use asap_types::ir::operator::{AtModifier, Source}; +use asap_types::ir::operator::Source; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::{NonASAPOp, Operator}; use asap_types::plan::physical_asap_dag::{ @@ -60,13 +66,13 @@ use asap_types::plan::physical_asap_dag::{ }; use asap_types::stages::{ ExecutionContext, Lifetime, Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, + PhysicalEdgeId, }; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::milp::model::QueryId; use crate::plan::candidate_node; use crate::retention::{edge_retention_need_ms, edge_scan_read_ms, output_retention_need_ms}; -use crate::selectable::{evaluation_grid, request_time_grid_ms, Execution, ExecutionContexts}; +use crate::selectable::{evaluation_grid, Execution, SelectableExecutions}; /// The objective's coefficients /// (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, @@ -74,18 +80,23 @@ use crate::selectable::{evaluation_grid, request_time_grid_ms, Execution, Execut pub(crate) struct ExecutionCosts { /// $\rho_\kappa c_{\kappa,v}$ of each execution, $W^{\mathrm{mat}}_v$ /// of an execution in `Materialized(v)`, by position in - /// `ExecutionContexts::executions`; or why `asap-cost` could not cost - /// it (the MILP leaves it out). + /// `SelectableExecutions::executions`; or why `asap-cost` could not + /// cost it (the MILP leaves it out). pub(crate) per_execution: Vec>, /// $\rho_\kappa c_{\kappa,u,c,i}$ of each edge, by position in - /// `ExecutionContexts::edges`; `None` when it could not be costed (the - /// MILP leaves it out). + /// `SelectableExecutions::edges`; `None` when it could not be costed + /// (the MILP leaves it out). pub(crate) per_edge: Vec>, - /// $\pi_v$, per ms retained, of each candidate that retains rows - /// (`PhysicalASAPDAGNode::retains_rows`) only within a bound: it has - /// $h_v$ and constraint (R). One whose consumer reads it unbounded is - /// costed retaining everything. - pub(crate) retention_cost_per_second: HashMap, + /// $\pi_v$, the cost per second of each second retained, of each + /// candidate that retains rows (`PhysicalASAPDAGNode::retains_rows`) + /// only within a bound: it has $h_v$ and constraint (R). One whose + /// consumer reads it unbounded is costed retaining everything. + pub(crate) retention_cost_per_retained_second: HashMap, + /// $\operatorname{need}_{\kappa,u,c,i}$ in ms of each edge, by position + /// in `SelectableExecutions::edges` ([`edge_retention_need_ms`]); + /// `None` when its producer retains no rows or the edge reads it + /// unbounded. + pub(crate) retention_need_ms_per_edge: Vec>, /// The source of each query-time `Scan`: its retention is raw data on /// disk, which a plan keeps once per source for the selected scan that /// keeps the most, as `plan_cost` charges it ($r_s$). @@ -93,28 +104,22 @@ pub(crate) struct ExecutionCosts { /// The cost per second of the raw data a query-time `Scan` without /// $h_v$ keeps on disk, retaining everything. pub(crate) unbounded_raw_data_on_disk_cost_per_second: HashMap, - /// The runtime's fixed overhead per query request, per second - /// ([`asap_cost::request_overhead_cost_per_second`]): every plan pays - /// it. - pub(crate) request_overhead_cost_per_second: f64, } -/// The objective's coefficients of every execution and edge of `contexts` -/// ([`ExecutionCosts`]; module doc). `query_outputs` are the executions -/// that may answer each query output. Errs on nothing: what `asap-cost` -/// cannot cost is reported in [`ExecutionCosts::per_execution`] or left -/// out of [`ExecutionCosts::per_edge`]. +/// The objective's coefficients of every execution and edge of +/// `selectable` ([`ExecutionCosts`]; module doc). Errs on nothing: what +/// `asap-cost` cannot cost is reported in [`ExecutionCosts::per_execution`] +/// or left out of [`ExecutionCosts::per_edge`]. pub(crate) fn execution_costs( physical: &PhysicalASAPMemoCandidates, - contexts: &ExecutionContexts, - query_outputs: &[(QueryId, Vec)], + selectable: &SelectableExecutions, entries: &[QueryWorkloadEntry], data: &DataWorkload, cost_inputs: &CostInputs, - request_overhead_cost_per_second: f64, ) -> ExecutionCosts { let memo = &physical.logical_candidates.memo; let count = physical.physical_candidates.len(); + let executions = selectable.executions(); // Each candidate's node, its inputs numbered 0.. (the ids do not // matter to `node_resource_usage_and_cost`), retaining nothing yet. let node_retaining = |v: PhysicalCandidateId, retention_ms: Option| { @@ -129,6 +134,24 @@ pub(crate) fn execution_costs( let query_time_scan: Vec = (0..count) .map(|v| is_query_time_scan(physical, PhysicalCandidateId(v))) .collect(); + let producer_of = |position: usize| { + let (_, edge, producer) = selectable.edges()[position]; + (edge, executions[producer].candidate) + }; + // Each edge's need, computed once per edge of the physical memo: an + // edge is read in every context it is selectable in. + let mut need_ms_of_edge: HashMap> = HashMap::new(); + let retention_need_ms_per_edge: Vec> = (0..selectable.edges().len()) + .map(|position| { + let (edge, u) = producer_of(position); + if !retains_rows[u.0] { + return None; + } + *need_ms_of_edge + .entry(edge) + .or_insert_with(|| edge_retention_need_ms(physical, entries, edge)) + }) + .collect(); // The most any consumer of each candidate that retains rows reads back // ($h_v$ at its largest): `None` when one reads it unbounded. let mut most_read_ms: HashMap> = (0..count) @@ -140,41 +163,64 @@ pub(crate) fn execution_costs( *most = most.zip(need_ms).map(|(most, need)| most.max(need)); } }; - for &(_, edge, producer) in contexts.edges() { - read( - producer.candidate, - edge_retention_need_ms(physical, entries, edge), - ); + for (position, &need_ms) in retention_need_ms_per_edge.iter().enumerate() { + read(producer_of(position).1, need_ms); } - for (query, executions) in query_outputs { - for execution in executions { + for (query, answering) in &selectable.answering { + for &index in answering { read( - execution.candidate, + executions[index].candidate, Some(output_retention_need_ms(&entries[query.0])), ); } } - // Each query-time `Scan`'s read per execution at its largest, and per - // execution: the most any consumer it may have in that context reads - // in one run (`None`: one reads it whole), charged once. - let mut scan_read_ms: HashMap> = (0..count) + // How far back a query-time `Scan` reads for the consumer of each edge + // out of it, in one run (`None`: read whole), once per memo edge. + let mut scan_read_ms_of_edge: HashMap> = HashMap::new(); + let scan_read_ms_per_edge: Vec>> = (0..selectable.edges().len()) + .map(|position| { + let (edge, u) = producer_of(position); + query_time_scan[u.0].then(|| { + *scan_read_ms_of_edge + .entry(edge) + .or_insert_with(|| edge_scan_read_ms(physical, edge)) + }) + }) + .collect(); + // Each query-time `Scan`'s read for its baseline output (what its + // consumers' baselines read): the most any of its consumers reads. + let mut baseline_scan_read_ms: HashMap> = (0..count) .filter(|&v| query_time_scan[v]) .map(|v| (PhysicalCandidateId(v), Some(0))) .collect(); - let mut execution_read_ms: Vec>> = vec![None; contexts.executions().len()]; - for &(_, edge, producer) in contexts.edges() { - let Some(most) = scan_read_ms.get_mut(&producer.candidate) else { + // A query-time `Scan`'s read per execution: one that is not + // materialized has one reader per context, and its read is on the edge + // to it (0 here); one materialized, shared by its readers, reads the + // most any of them reads in one run. + let mut execution_read_ms: Vec> = executions + .iter() + .map(|execution| query_time_scan[execution.candidate.0].then_some(0)) + .collect(); + for (position, read_ms) in scan_read_ms_per_edge.iter().enumerate() { + let Some(read_ms) = *read_ms else { continue; }; - let read_ms = edge_scan_read_ms(physical, edge); + let (_, _, producer) = selectable.edges()[position]; + let u = executions[producer].candidate; + let most = baseline_scan_read_ms + .get_mut(&u) + .expect("a query-time scan"); *most = most.zip(read_ms).map(|(most, read)| most.max(read)); - let of_execution = &mut execution_read_ms[contexts.index_of(producer)]; - *of_execution = Some(match *of_execution { - None => read_ms, - Some(most) => most.zip(read_ms).map(|(most, read)| most.max(read)), - }); + if physical.physical_candidates[u.0] + .physical_properties + .is_materialized() + { + let of_execution = &mut execution_read_ms[producer]; + *of_execution = of_execution.zip(read_ms).map(|(most, read)| most.max(read)); + } } - let scan_read_of = |v: PhysicalCandidateId| scan_read_ms.get(&v).copied().flatten(); + let baseline_scan_read_of = + |v: PhysicalCandidateId| baseline_scan_read_ms.get(&v).copied().flatten(); let nodes: Vec = (0..count) .map(PhysicalCandidateId) .map(|v| node_retaining(v, most_read_ms.get(&v).copied().flatten())) @@ -213,11 +259,12 @@ pub(crate) fn execution_costs( ) .and_then(|usage| cost_per_second(&usage, cost_inputs)) }; - let cost = - |v: PhysicalCandidateId, - node: &PhysicalASAPDAGNode, - inputs: &[ProducerOutput<'_>], - per_second: f64| { cost_reading(v, node, inputs, per_second, scan_read_of(v)) }; + let cost = |v: PhysicalCandidateId, + node: &PhysicalASAPDAGNode, + inputs: &[ProducerOutput<'_>], + per_second: f64| { + cost_reading(v, node, inputs, per_second, baseline_scan_read_of(v)) + }; // Each candidate's usage of one computation (what it outputs does not // depend on how often it runs) over its baseline inputs. let mut usage: Vec, String>>> = @@ -233,7 +280,7 @@ pub(crate) fn execution_costs( &nodes[v.0], inputs, 1.0, - scan_read_of(v), + baseline_scan_read_of(v), data, cost_inputs, ), @@ -243,11 +290,10 @@ pub(crate) fn execution_costs( } // Each execution's (own, raw data on disk) cost per second at its // rate, over its baseline inputs, and that rate. - let at_rate: Vec> = contexts - .executions() + let at_rate: Vec> = executions .iter() .zip(&execution_read_ms) - .map(|(&execution, read_ms)| { + .map(|(&execution, &read_ms)| { let v = execution.candidate; if let Some(Err(error)) = &usage[v.0] { return Err(error.clone()); @@ -255,32 +301,32 @@ pub(crate) fn execution_costs( let per_second = executions_per_second( physical, entries, - contexts, + selectable, execution, cost_inputs.non_repeating_query_horizon_seconds, )?; let inputs = baseline[v.0] .as_ref() .ok_or("it reads a group on a cycle of candidates")?; - let read_ms = read_ms.unwrap_or_else(|| scan_read_of(v)); let (own, raw_data) = cost_reading(v, &nodes[v.0], inputs, per_second, read_ms)?; Ok((own, raw_data, per_second)) }) .collect(); - // The one execution of each materialized candidate, by position. - let own_execution: HashMap = contexts - .executions() + // The one execution of each candidate materialized across evaluations, + // by position. + let own_execution: HashMap = executions .iter() .enumerate() - .filter(|(_, execution)| materialized(physical, execution.candidate)) + .filter(|(_, execution)| is_across_evaluations(physical, execution.candidate)) .map(|(index, execution)| (execution.candidate, index)) .collect(); // π_v of a candidate retaining rows every consumer reads within a bound: - // what one more Δ ms of retention costs, over Δ (a window of a chain, - // else a second), at its retention's largest; its whole own cost when - // it is materialized, its raw data on disk for a query-time scan (each - // of its runs reads what it reads at that retention, ρ_κ c_{κ,v}). - let mut retention_cost_per_second = HashMap::new(); + // what one more Δ ms of retention costs, per second retained (Δ a + // window of a chain, else a second), at its retention's largest; its + // whole own cost when it is materialized, its raw data on disk for a + // query-time scan (each of its runs reads what it reads at that + // retention, ρ_κ c_{κ,v}). + let mut retention_cost_per_retained_second = HashMap::new(); for (&v, &most_read_ms) in &most_read_ms { let Some(retention_ms) = most_read_ms else { continue; @@ -290,12 +336,13 @@ pub(crate) fn execution_costs( .unwrap_or(1_000) .max(1); let longer = node_retaining(v, Some(retention_ms + delta_ms)); - let per_ms = match materialized(physical, v) { + let delta_seconds = delta_ms as f64 / 1_000.0; + let per_retained_second = match is_across_evaluations(physical, v) { true => own_execution.get(&v).and_then(|&index| { let (own, raw_data, per_second) = at_rate[index].as_ref().ok()?; let inputs = baseline[v.0].as_ref()?; let (longer_own, longer_raw_data) = cost(v, &longer, inputs, *per_second).ok()?; - Some((longer_own + longer_raw_data - own - raw_data) / delta_ms as f64) + Some((longer_own + longer_raw_data - own - raw_data) / delta_seconds) }), false => usage[v.0] .as_ref() @@ -304,11 +351,11 @@ pub(crate) fn execution_costs( let inputs = baseline[v.0].as_ref()?; let (_, raw_data) = cost(v, &nodes[v.0], inputs, 1.0).ok()?; let (_, longer_raw_data) = cost(v, &longer, inputs, 1.0).ok()?; - Some((longer_raw_data - raw_data) / delta_ms as f64) + Some((longer_raw_data - raw_data) / delta_seconds) }), }; - if let Some(per_ms) = per_ms { - retention_cost_per_second.insert(v, per_ms.max(0.0)); + if let Some(per_retained_second) = per_retained_second { + retention_cost_per_retained_second.insert(v, per_retained_second.max(0.0)); } } // Each query-time scan's source and, without $h_v$, the raw data it @@ -326,7 +373,7 @@ pub(crate) fn execution_costs( continue; }; raw_data_source.insert(v, source.clone()); - if !retention_cost_per_second.contains_key(&v) { + if !retention_cost_per_retained_second.contains_key(&v) { if let Some(Ok((_, raw_data))) = baseline[v.0] .as_ref() .map(|inputs| cost(v, &nodes[v.0], inputs, 1.0)) @@ -335,33 +382,32 @@ pub(crate) fn execution_costs( } } } - let per_execution: Vec> = contexts - .executions() + let per_execution: Vec> = executions .iter() .zip(&at_rate) .map(|(execution, at_rate)| { let &(own, raw_data, _) = at_rate.as_ref().map_err(Clone::clone)?; let v = execution.candidate; - Ok(match retention_cost_per_second.get(&v) { + Ok(match retention_cost_per_retained_second.get(&v) { // A query-time scan's raw data is charged per source. _ if query_time_scan[v.0] => own, - Some(per_ms) if materialized(physical, v) => { - let retention_ms = most_read_ms[&v].unwrap_or(0) as f64; - own + raw_data - per_ms * retention_ms + Some(per_retained_second) if is_across_evaluations(physical, v) => { + let retention_seconds = most_read_ms[&v].unwrap_or(0) as f64 / 1_000.0; + own + raw_data - per_retained_second * retention_seconds } _ => own + raw_data, }) }) .collect(); - let per_edge = contexts + let per_edge = selectable .edges() .iter() - .map(|&(consumer, edge, producer)| { + .zip(&scan_read_ms_per_edge) + .map(|(&(consumer, edge, producer), scan_read_ms)| { let e = physical.edge(edge); let (u, c) = (e.producer_candidate_id, e.consumer_candidate_id); - per_execution[contexts.index_of(producer)].as_ref().ok()?; - let &(own, raw_data, per_second) = - at_rate[contexts.index_of(consumer)].as_ref().ok()?; + per_execution[producer].as_ref().ok()?; + let &(own, raw_data, per_second) = at_rate[consumer].as_ref().ok()?; let produced = usage[u.0].as_ref()?.as_ref().ok()?; let mut inputs = baseline[c.0].clone()?; if !reads_one_batch_of(&nodes, c, u) { @@ -369,27 +415,41 @@ pub(crate) fn execution_costs( produced.producer_output(&nodes[u.0].payload, &nodes[u.0].output_schema); } let (with_u_own, with_u_raw_data) = cost(c, &nodes[c.0], &inputs, per_second).ok()?; - Some(with_u_own + with_u_raw_data - own - raw_data) + // A scan that is not materialized reads, in its one consumer's + // run, what that consumer reads: on this edge. + let scan_read = match scan_read_ms { + Some(read_ms) if nodes[u.0].lifetime == Lifetime::None => { + let &(scan_own, _, scan_per_second) = at_rate[producer].as_ref().ok()?; + let (reading_own, _) = cost_reading( + u, + &nodes[u.0], + baseline[u.0].as_ref()?, + scan_per_second, + *read_ms, + ) + .ok()?; + reading_own - scan_own + } + _ => 0.0, + }; + Some(with_u_own + with_u_raw_data - own - raw_data + scan_read) }) .collect(); ExecutionCosts { per_execution, per_edge, - retention_cost_per_second, + retention_cost_per_retained_second, + retention_need_ms_per_edge, raw_data_source, unbounded_raw_data_on_disk_cost_per_second, - request_overhead_cost_per_second, } } /// Whether candidate `v` is materialized across evaluations. -fn materialized(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { - matches!( - physical.physical_candidates[v.0] - .physical_properties - .lifetime(), - Lifetime::AcrossEvaluations(_) - ) +fn is_across_evaluations(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { + physical.physical_candidates[v.0] + .physical_properties + .is_across_evaluations() } /// Whether candidate `v` is a `Scan` at query time, which reads raw data @@ -409,27 +469,24 @@ fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidat } /// $\rho$ of `execution`'s context, how often it runs, from the queries it -/// serves: an ingestion-time execution once per second of arriving rows; -/// in an evaluation context, the combined evaluation rate -/// ([`evaluation_rate`]) of the queries on its grid that reach its group, -/// with their range evaluations, and of the range queries whose request -/// end (start) times are the grid, once per request (an input of a -/// `TimeShift` anchored with `@`); in `Materialized(m)`, as often as $m$ -/// runs ([`asap_cost::plan::execution_demand::executions_per_second`]). +/// serves: in an evaluation context, as often as that context runs for +/// the queries on its grid that reach the execution's group +/// ([`context_executions_per_second`], which `plan_cost` uses too): at +/// their evaluations, or, for the inputs of a `TimeShift` anchored with +/// `@`, once per request of the queries on the anchored evaluation grid; +/// in `Materialized(m)`, as often as $m$ runs +/// ([`asap_cost::plan::execution_demand::executions_per_second`]); in the +/// ingestion pipeline, once per second of arriving rows. /// /// Example: queries every 10 s, 10 s and 30 s reaching a scan: 1/10 per /// second in the 10 s context, 1/30 in the 30 s one. pub(crate) fn executions_per_second( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], - contexts: &ExecutionContexts, + selectable: &SelectableExecutions, execution: Execution, non_repeating_query_horizon_seconds: f64, ) -> Result { - let candidate = &physical.physical_candidates[execution.candidate.0]; - if candidate.physical_properties.timing() == ExecutionTiming::IngestionTime { - return Ok(1.0); - } let memo = &physical.logical_candidates.memo; let reaching = |v: PhysicalCandidateId| { let group = physical.physical_candidates[v.0].logical_candidate_id.group; @@ -438,30 +495,29 @@ pub(crate) fn executions_per_second( .iter() .filter_map(|&query| entries.get(query)) }; - let rate = match contexts.context(execution.context) { - ExecutionContext::Evaluation { grid } => evaluation_rate( - reaching(execution.candidate).filter_map(|entry| { - if evaluation_grid(entry) == grid { - return Some((&entry.recurrence, entry.range_evaluation)); - } - let request_time = [AtModifier::End, AtModifier::Start] - .into_iter() - .any(|at| grid.period_ms == request_time_grid_ms(entry, at)); - (entry.range_evaluation.is_some() && request_time) - .then_some((&entry.recurrence, None)) - }), - non_repeating_query_horizon_seconds, - ), + let rate = match selectable.context(execution.context) { + ExecutionContext::Evaluation { grid } => { + let (anchor, queries_grid) = match selectable.request_anchor(execution.context) { + Some((at, queries_grid)) => (Some(at), queries_grid), + None => (None, grid), + }; + context_executions_per_second( + anchor, + reaching(execution.candidate) + .filter(|entry| evaluation_grid(entry) == queries_grid), + non_repeating_query_horizon_seconds, + ) + } ExecutionContext::Materialized(m) => { let properties = physical.physical_candidates[m.0].physical_properties; - rate_of( + rate_of_materialized( properties.timing(), properties.lifetime(), reaching(m), non_repeating_query_horizon_seconds, ) } - // Only ingestion-time candidates run in an ingestion pipeline. + // Each arriving row once. ExecutionContext::IngestionPipeline(_) => Ok(1.0), }; rate.map_err(|error| error.to_string()) diff --git a/crates/stage3-plan-selection/src/cost/tests.rs b/crates/stage3-plan-selection/src/cost/tests.rs index e6c8a84dd..ba0012f4d 100644 --- a/crates/stage3-plan-selection/src/cost/tests.rs +++ b/crates/stage3-plan-selection/src/cost/tests.rs @@ -19,7 +19,7 @@ use asap_types::workload::{ use super::*; use crate::retention::edge_retention_need_ms; -use crate::selectable::ExecutionContexts; +use crate::selectable::{QueryId, SelectableExecutions}; const DAY_MS: u64 = 86_400_000; @@ -172,50 +172,48 @@ fn no_execution_reading_a_producer_costs_less_than_nothing() { "sources": {"m": {"series_count": 1_000_000, "sample_interval": 15_000}} })) .unwrap(); - let (contexts, query_outputs) = ExecutionContexts::new( + let selectable = SelectableExecutions::new( &physical, &entries, &[(QueryId(0), state), (QueryId(1), newest_day)], ); let costs = execution_costs( &physical, - &contexts, - &query_outputs, + &selectable, &entries, &data, &CostInputs::default(), - 0.0, ); // π_v times the least any consumer of `v` reads back, the least // retention (R) lets a plan selecting `v` keep. let least_retention = |v: PhysicalCandidateId| { - let least_need_ms = contexts + let least_need_ms = selectable .edges() .iter() - .filter(|(_, _, producer)| producer.candidate == v) + .filter(|&&(_, _, producer)| selectable.executions()[producer].candidate == v) .filter_map(|&(_, edge, _)| edge_retention_need_ms(&physical, &entries, edge)) .min() .unwrap_or(0); costs - .retention_cost_per_second + .retention_cost_per_retained_second .get(&v) .copied() .unwrap_or(0.0) * least_need_ms as f64 + / 1_000.0 }; let mut checked = 0; - for (index, &(consumer, edge, _)) in contexts.edges().iter().enumerate() { - let (Ok(execution), Some(edge_cost)) = ( - &costs.per_execution[contexts.index_of(consumer)], - costs.per_edge[index], - ) else { + for (index, &(consumer, edge, _)) in selectable.edges().iter().enumerate() { + let (Ok(execution), Some(edge_cost)) = + (&costs.per_execution[consumer], costs.per_edge[index]) + else { continue; }; - let total = execution + edge_cost + least_retention(consumer.candidate); + let consumer = selectable.executions()[consumer].candidate; + let total = execution + edge_cost + least_retention(consumer); assert!( total >= 0.0, - "{:?} reading through {edge:?}: {execution} + {edge_cost} + retention = {total}", - consumer.candidate + "{consumer:?} reading through {edge:?}: {execution} + {edge_cost} + retention = {total}" ); checked += 1; } diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index fc9c98b01..83941592c 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -34,10 +34,11 @@ use asap_types::stages::{PhysicalASAPMemoCandidates, PruneReason, PrunedCandidat use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::cost::execution_costs; -use crate::milp::model::{solve_selection, QueryId, SelectionInputs, SelectionMilpError}; +use crate::milp::exact_only::must_not_read; +use crate::milp::model::{solve_selection, SelectionInputs, SelectionMilpError}; use crate::plan::plan_of; use crate::report::{accepted_accuracy, infeasible}; -use crate::selectable::{output_groups, Execution, ExecutionContexts}; +use crate::selectable::{output_groups, SelectableExecutions}; pub use report::{ large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, @@ -77,33 +78,21 @@ pub fn select_plan( }); } let outputs = output_groups(physical); - let (contexts, output_executions) = ExecutionContexts::new(physical, entries, &outputs); + let mut selectable = SelectableExecutions::new(physical, entries, &outputs); let request_overhead_cost_per_second = request_overhead_cost_per_second(entries, cost_inputs) .map_err(|error| ErrorSelectingPlan::Cost { candidate: "every plan (the request overhead)".into(), node: 0, error, })?; - let costs = execution_costs( - physical, - &contexts, - &output_executions, - entries, - data, - cost_inputs, - request_overhead_cost_per_second, - ); - let query_outputs: Vec<(QueryId, Vec)> = output_executions - .into_iter() - .map(|(query, executions)| { - let costed = executions - .into_iter() - .filter(|&execution| costs.per_execution[contexts.index_of(execution)].is_ok()) - .collect(); - (query, costed) - }) - .collect(); - if query_outputs + let costs = execution_costs(physical, &selectable, entries, data, cost_inputs); + // A query output is answered by an execution that could be costed and + // that its query may read (X). + selectable.retain_answering(|query, index, execution| { + costs.per_execution[index].is_ok() && !must_not_read(physical, query, execution.candidate) + }); + if selectable + .answering .iter() .any(|(_, executions)| executions.is_empty()) { @@ -112,9 +101,8 @@ pub fn select_plan( let solved = solve_selection(&SelectionInputs { physical, entries, - contexts: &contexts, + selectable: &selectable, costs: &costs, - query_outputs: &query_outputs, }) .map_err(|error| match error { SelectionMilpError::Infeasible => infeasible(&costs), @@ -125,7 +113,7 @@ pub fn select_plan( })?; // A candidate none of whose executions could be costed. let mut costed = vec![None; physical.physical_candidates.len()]; - for (execution, cost) in contexts.executions().iter().zip(&costs.per_execution) { + for (execution, cost) in selectable.executions().iter().zip(&costs.per_execution) { let known = costed[execution.candidate.0].get_or_insert(false); *known |= cost.is_ok(); } @@ -140,7 +128,8 @@ pub fn select_plan( reason: PruneReason::UsageNotCostable, }) .collect(), - selected_plan_cost_per_second: solved.objective_cost_per_second, + selected_plan_cost_per_second: solved.cost_per_second_without_request_overhead + + request_overhead_cost_per_second, milp_variable_count: solved.variable_count, milp_constraint_count: solved.constraint_count, milp_construction_ms: solved.construction_ms, diff --git a/crates/stage3-plan-selection/src/milp/exact_only.rs b/crates/stage3-plan-selection/src/milp/exact_only.rs index 8121db80b..ab34f582f 100644 --- a/crates/stage3-plan-selection/src/milp/exact_only.rs +++ b/crates/stage3-plan-selection/src/milp/exact_only.rs @@ -13,25 +13,26 @@ use std::collections::{BTreeMap, BTreeSet}; use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; -use super::model::{QueryId, SelectionInputs}; -use crate::selectable::Execution; +use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId}; -/// Whether query `query` must not read `execution`: its candidate is not -/// exact (it has sketch parameters θ) and its group is one the query may -/// read only exactly. A readout's own candidate has no θ, but the sketch -/// states it reads are in the same list, as every group below a listed one -/// is. +use super::model::SelectionInputs; +use crate::selectable::QueryId; + +/// Whether query `query` must not read candidate `v`: it is not exact (it +/// has sketch parameters θ) and its group is one the query may read only +/// exactly. A readout's own candidate has no θ, but the sketch states it +/// reads are in the same list, as every group below a listed one is. /// /// Example: `quantile_over_time(0.9, m[5m])` at a relative target lists /// the quantile and every group below it; its KLL state builds (θ) are -/// executions it must not read, the exact quantile one it may. +/// candidates it must not read, the exact quantile one it may. pub(crate) fn must_not_read( - inputs: &SelectionInputs<'_>, + physical: &PhysicalASAPMemoCandidates, query: QueryId, - execution: Execution, + v: PhysicalCandidateId, ) -> bool { - let logical = &inputs.physical.logical_candidates; - let id = inputs.physical.physical_candidates[execution.candidate.0].logical_candidate_id; + let logical = &physical.logical_candidates; + let id = physical.physical_candidates[v.0].logical_candidate_id; logical .exact_only_groups_of_query .get(query.0) @@ -45,36 +46,37 @@ pub(crate) fn must_not_read( } /// $a_{q,e}$, added to `problem`, for each query $q$ and each execution $e$ -/// it may reach from its output executions over selectable edges (`y`) -/// from which an execution it must not read ([`must_not_read`]) is -/// reachable, less those themselves. None for a query with no exact-only -/// group. +/// (by position in `SelectableExecutions::executions`) it may reach from +/// its output executions over selectable edges (`y`) from which an +/// execution it must not read ([`must_not_read`]) is reachable, less those +/// themselves. None for a query with no exact-only group. pub(crate) fn exact_only_variables( inputs: &SelectionInputs<'_>, y: &BTreeMap, problem: &mut ProblemVariables, -) -> BTreeMap<(QueryId, Execution), Variable> { - let contexts = inputs.contexts; - let count = contexts.executions().len(); +) -> BTreeMap<(QueryId, usize), Variable> { + let selectable = inputs.selectable; + let count = selectable.executions().len(); // The selectable edges, as producers per consumer and back. let mut producers = vec![Vec::new(); count]; let mut consumers = vec![Vec::new(); count]; for &position in y.keys() { - let (consumer, _, producer) = contexts.edges()[position]; - let (c, u) = (contexts.index_of(consumer), contexts.index_of(producer)); + let (c, _, u) = selectable.edges()[position]; producers[c].push(u); consumers[u].push(c); } - let execution = |e: usize| contexts.executions()[e]; - let queries: BTreeSet = inputs.query_outputs.iter().map(|(q, _)| *q).collect(); + let forbidden = |query: QueryId, e: usize| { + must_not_read(inputs.physical, query, selectable.executions()[e].candidate) + }; + let queries: BTreeSet = selectable.answering.iter().map(|(q, _)| *q).collect(); let mut tracked = BTreeMap::new(); for query in queries { let mut reached = vec![false; count]; - let mut pending: Vec = inputs - .query_outputs + let mut pending: Vec = selectable + .answering .iter() .filter(|(q, _)| *q == query) - .flat_map(|(_, executions)| executions.iter().map(|&e| contexts.index_of(e))) + .flat_map(|(_, executions)| executions.iter().copied()) .collect(); while let Some(e) = pending.pop() { if !std::mem::replace(&mut reached[e], true) { @@ -83,17 +85,15 @@ pub(crate) fn exact_only_variables( } let mut reaches_forbidden = vec![false; count]; let mut pending: Vec = (0..count) - .filter(|&e| reached[e] && must_not_read(inputs, query, execution(e))) + .filter(|&e| reached[e] && forbidden(query, e)) .collect(); while let Some(e) = pending.pop() { if reached[e] && !std::mem::replace(&mut reaches_forbidden[e], true) { pending.extend(&consumers[e]); } } - for e in (0..count) - .filter(|&e| reaches_forbidden[e] && !must_not_read(inputs, query, execution(e))) - { - tracked.insert((query, execution(e)), problem.add(variable().min(0).max(1))); + for e in (0..count).filter(|&e| reaches_forbidden[e] && !forbidden(query, e)) { + tracked.insert((query, e), problem.add(variable().min(0).max(1))); } } tracked @@ -102,34 +102,33 @@ pub(crate) fn exact_only_variables( /// (X) Query $q$ reads the groups it may read only exactly through exact /// candidates: $a_{q,e} \ge z_{q,e}$ for an output execution, /// $a_{q,p} \ge y_{\kappa,p,c,i} + a_{q,c} - 1$ along a selectable edge, so -/// $a_{q,e} = 1$ on every execution $q$'s selected path reads; an execution -/// $q$ must not read ([`must_not_read`]) has $a_{q,e} = 0$: $z_{q,e} = 0$ -/// and $y_{\kappa,p,c,i} + a_{q,c} \le 1$. Other queries may still read it, -/// through their own paths. $a$ exists only where [`exact_only_variables`] -/// tracks it, and is not in the objective. +/// $a_{q,e} = 1$ on every execution $q$'s selected path reads; along an +/// edge into an execution $p$ $q$ must not read ([`must_not_read`]), +/// $y_{\kappa,p,c,i} + a_{q,c} \le 1$. No $z_{q,e}$ exists for an output +/// execution $q$ must not read (`crate::select_plan` leaves it out). Other +/// queries may still read it, through their own paths. $a$ exists only +/// where [`exact_only_variables`] tracks it, and is not in the objective. pub(crate) fn exact_only_constraints( inputs: &SelectionInputs<'_>, - a: &BTreeMap<(QueryId, Execution), Variable>, + a: &BTreeMap<(QueryId, usize), Variable>, y: &BTreeMap, - z: &BTreeMap<(QueryId, Execution), Variable>, + z: &BTreeMap<(QueryId, usize), Variable>, ) -> Vec { - let mut constraints = Vec::new(); - for (&(query, execution), &z) in z { - if must_not_read(inputs, query, execution) { - constraints.push(constraint!(z <= 0)); - } else if let Some(&a) = a.get(&(query, execution)) { - constraints.push(constraint!(a >= z)); - } - } + let selectable = inputs.selectable; + let mut constraints: Vec = z + .iter() + .filter_map(|(key, &z)| Some(constraint!(*a.get(key)? >= z))) + .collect(); // The tracked queries, per execution. - let mut readers: BTreeMap> = BTreeMap::new(); + let mut readers: BTreeMap> = BTreeMap::new(); for (&(query, execution), &a) in a { readers.entry(execution).or_default().push((query, a)); } for (&position, &y) in y { - let (consumer, _, producer) = inputs.contexts.edges()[position]; + let (consumer, _, producer) = selectable.edges()[position]; for &(query, a_consumer) in readers.get(&consumer).into_iter().flatten() { - if must_not_read(inputs, query, producer) { + let v = selectable.executions()[producer].candidate; + if must_not_read(inputs.physical, query, v) { constraints.push(constraint!(y + a_consumer <= 1)); } else if let Some(&a_producer) = a.get(&(query, producer)) { constraints.push(constraint!(a_producer >= y + a_consumer - 1)); @@ -138,21 +137,3 @@ pub(crate) fn exact_only_constraints( } constraints } - -/// Per execution, by position in `ExecutionContexts::executions`, whether -/// some query must not read it ([`must_not_read`]): the greedy bound -/// ([`super::upper_bound`]) leaves them out, so the plan it finds meets (X) -/// whichever query reads what. -pub(crate) fn not_read_by_some_query(inputs: &SelectionInputs<'_>) -> Vec { - let queries: BTreeSet = inputs.query_outputs.iter().map(|(q, _)| *q).collect(); - inputs - .contexts - .executions() - .iter() - .map(|&execution| { - queries - .iter() - .any(|&query| must_not_read(inputs, query, execution)) - }) - .collect() -} diff --git a/crates/stage3-plan-selection/src/milp/execution_contexts.rs b/crates/stage3-plan-selection/src/milp/execution_contexts.rs deleted file mode 100644 index 3e4d7b64a..000000000 --- a/crates/stage3-plan-selection/src/milp/execution_contexts.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! The execution-context extension of the selection MILP -//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, -//! section 5): the retention amount $h_v$ and constraint (R). The -//! executions $x_{\kappa,v}$ and edges $y_{\kappa,u,c,i}$ it selects over -//! are [`crate::selectable::ExecutionContexts`]'s, in [`super::model`]. - -use std::collections::{BTreeMap, HashMap}; - -use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; - -use asap_types::stages::PhysicalCandidateId; - -use super::model::{QueryId, SelectionInputs}; -use crate::retention::{edge_retention_need_ms, output_retention_need_ms}; -use crate::selectable::Execution; - -/// $h_v$ in seconds (the unit keeps (R)'s coefficients near the others'), -/// for each candidate with a retention cost -/// ([`crate::cost::ExecutionCosts::retention_cost_per_second`]), added to -/// `problem`. -pub(crate) fn retention_variables( - inputs: &SelectionInputs<'_>, - problem: &mut ProblemVariables, -) -> BTreeMap { - let mut retaining: Vec = inputs - .costs - .retention_cost_per_second - .keys() - .copied() - .collect(); - retaining.sort_unstable(); - retaining - .into_iter() - .map(|v| (v, problem.add(variable().min(0)))) - .collect() -} - -/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$ and -/// $h_v \ge \operatorname{span}_q\, z_{q,v}$, for a candidate $v$ with $h_v$: -/// how far back its rows are read through each selectable edge `y` (the -/// reading path's offset plus range, [`edge_retention_need_ms`]) and by -/// each query reading it directly (its range-evaluation span), a constant -/// per edge, so (R) is linear. Query latency is ignored: a result counts as -/// read when its evaluation starts. -pub(crate) fn retention_constraints( - inputs: &SelectionInputs<'_>, - h: &BTreeMap, - y: &BTreeMap, - z: &BTreeMap<(QueryId, Execution), Variable>, -) -> Vec { - let mut constraints = Vec::new(); - for (&position, &y) in y { - let (_, edge, producer) = inputs.contexts.edges()[position]; - let Some(&h) = h.get(&producer.candidate) else { - continue; - }; - let need_ms = edge_retention_need_ms(inputs.physical, inputs.entries, edge) - .expect("a candidate with h_v is read within a bound"); - constraints.push(constraint!(h >= seconds(need_ms) * y)); - } - for (&(query, execution), &z) in z { - if let Some(&h) = h.get(&execution.candidate) { - let need_ms = output_retention_need_ms(&inputs.entries[query.0]); - constraints.push(constraint!(h >= seconds(need_ms) * z)); - } - } - constraints -} - -/// The least $h_v$ (R) allows for a selection, in ms, of each candidate -/// with $h_v$ that `edges` (selected, by position in -/// `ExecutionContexts::edges`) or `outputs` (the query outputs and the -/// executions answering them) read: what the MILP's optimum retains when -/// $\pi_v > 0$, and what the plan retains. -pub(crate) fn least_retention_ms( - inputs: &SelectionInputs<'_>, - edges: impl IntoIterator, - outputs: impl IntoIterator, -) -> HashMap { - let retaining = &inputs.costs.retention_cost_per_second; - let mut retention_ms: HashMap = HashMap::new(); - let mut read = |v: PhysicalCandidateId, need_ms: u64| { - if retaining.contains_key(&v) { - let most = retention_ms.entry(v).or_default(); - *most = (*most).max(need_ms); - } - }; - for position in edges { - let (_, edge, producer) = inputs.contexts.edges()[position]; - if retaining.contains_key(&producer.candidate) { - let need_ms = edge_retention_need_ms(inputs.physical, inputs.entries, edge) - .expect("a candidate with h_v is read within a bound"); - read(producer.candidate, need_ms); - } - } - for (query, execution) in outputs { - read( - execution.candidate, - output_retention_need_ms(&inputs.entries[query.0]), - ); - } - retention_ms -} - -/// `ms` in seconds, the unit of $h_v$ in the MILP. -fn seconds(ms: u64) -> f64 { - ms as f64 / 1_000.0 -} diff --git a/crates/stage3-plan-selection/src/milp/graph.rs b/crates/stage3-plan-selection/src/milp/graph.rs index cc491999b..db9378d9e 100644 --- a/crates/stage3-plan-selection/src/milp/graph.rs +++ b/crates/stage3-plan-selection/src/milp/graph.rs @@ -8,7 +8,7 @@ use super::model::SelectionInputs; /// Each execution's strongly connected component of the selectable edges /// (those [`crate::cost::ExecutionCosts::per_edge`] costs), computed once -/// per model, by position in `ExecutionContexts::executions`. +/// per model, by position in `SelectableExecutions::executions`. pub(crate) struct Components { /// Each execution's component number. pub(crate) of: Vec, @@ -19,17 +19,15 @@ pub(crate) struct Components { impl Components { pub(crate) fn of(inputs: &SelectionInputs<'_>) -> Self { - let contexts = inputs.contexts; + let selectable = inputs.selectable; let of = strongly_connected_components( - contexts.executions().len(), - contexts + selectable.executions().len(), + selectable .edges() .iter() .zip(&inputs.costs.per_edge) .filter(|(_, cost)| cost.is_some()) - .map(|(&(consumer, _, producer), _)| { - (contexts.index_of(producer), contexts.index_of(consumer)) - }), + .map(|(&(consumer, _, producer), _)| (producer, consumer)), ); let mut count: HashMap = HashMap::new(); for &component in &of { diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index f0b474911..1a6d91f90 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -2,13 +2,13 @@ //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! sections 3–5): its variables over execution contexts and one function //! per constraint family ([`model`]), the retention amount $h_v$ and -//! constraint (R) ([`execution_contexts`]), each query's exact-only reads, +//! constraint (R) ([`retention`]), each query's exact-only reads, //! constraint (X) ([`exact_only`]), the graph algorithm it needs //! ([`graph`]), the bound it scales by ([`upper_bound`]), and the accuracy //! extension ([`accuracy`]). mod accuracy; -mod exact_only; -mod execution_contexts; +pub(crate) mod exact_only; mod graph; pub(crate) mod model; +mod retention; mod upper_bound; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 0a5826d75..d21477e33 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -5,7 +5,7 @@ //! | Family | Variables | Constraints | //! |---|---|---| //! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_constraints`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | -//! | retention (section 5) | $h_v$ | (R) [`super::execution_contexts::retention_constraints`] | +//! | retention (section 5) | $h_v$ | (R) [`super::retention::retention_constraints`] | //! | exact-only reads per query | $a_{q,e}$ | (X) [`super::exact_only::exact_only_constraints`] | //! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ (`AccuracyVariables`) | (K) `accuracy_constraints` | //! @@ -24,54 +24,47 @@ use good_lp::{ }; use asap_types::ir::operator::Source; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::{ - Lifetime, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, -}; +use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; use super::exact_only::{exact_only_constraints, exact_only_variables}; -use super::execution_contexts::{least_retention_ms, retention_constraints, retention_variables}; use super::graph::Components; +use super::retention::{least_retention_ms, retention_constraints, retention_variables}; use super::upper_bound::dearer_than_a_feasible_plan; use crate::cost::ExecutionCosts; -use crate::selectable::{Execution, ExecutionContexts}; - -/// A query, by index in the workload. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct QueryId(pub usize); +use crate::selectable::{has_one_reader_per_context, Execution, QueryId, SelectableExecutions}; /// What the MILP is built over. pub(crate) struct SelectionInputs<'a> { pub(crate) physical: &'a PhysicalASAPMemoCandidates, /// Each query's workload entry, for the retention needs of (R). pub(crate) entries: &'a [QueryWorkloadEntry], - /// The executions $x$ and edges $y$ are over. - pub(crate) contexts: &'a ExecutionContexts, + /// The executions $x$, edges $y$ and query outputs $z$ are over: + /// $z_{q,v}$ exists only for the executions + /// `SelectableExecutions::answering` lists. + pub(crate) selectable: &'a SelectableExecutions, /// The objective's coefficients: $x$ is 0 for an execution it could /// not cost, $y$ exists only on an edge it costs, $h_v$ only for a /// candidate with a retention cost. pub(crate) costs: &'a ExecutionCosts, - /// Each query output reference (a query, or one operator a scalar query - /// reads), with the executions that may answer it: $z_{q,v}$ exists - /// only for them. - pub(crate) query_outputs: &'a [(QueryId, Vec)], } -/// The variables (sections 3 and 5). +/// The variables (sections 3 and 5). Executions are by position in +/// `SelectableExecutions::executions`, edges by position in +/// `SelectableExecutions::edges`. pub(crate) struct SelectionVariables { - /// $x_{\kappa,v}$, by position in `ExecutionContexts::executions`. + /// $x_{\kappa,v}$, by execution. x: Vec, - /// $y_{\kappa,u,c,i}$, by position in `ExecutionContexts::edges`, on - /// the edges costed between executions kept, in edge order. + /// $y_{\kappa,u,c,i}$, by edge, on the edges costed between executions + /// kept, in edge order. y: BTreeMap, - /// $z_{q,v}$, by the execution answering $q$, in key order. - z: BTreeMap<(QueryId, Execution), Variable>, - /// $t$, by position in `ExecutionContexts::executions`: only for - /// executions on a cycle of selectable edges (a strongly connected - /// component); `None` elsewhere. + /// $z_{q,v}$, by query and the execution answering it, in key order. + z: BTreeMap<(QueryId, usize), Variable>, + /// $t$, by execution: only for executions on a cycle of selectable + /// edges (a strongly connected component); `None` elsewhere. t: Vec>, - /// $h_v$ in seconds, for a candidate with a retention cost. + /// $h_v$ in seconds retained, for a candidate with a retention cost and + /// an execution kept ([`retention_variables`]). h: BTreeMap, /// Per raw-data source a query-time scan keeps on disk at a cost: the /// cost per second of keeping it, at least each selected scan's @@ -79,43 +72,38 @@ pub(crate) struct SelectionVariables { raw_data_on_disk: BTreeMap, /// $a_{q,e}$: query $q$'s selected path reads execution $e$, where /// (X) needs it ([`exact_only_variables`]). - read_by_query: BTreeMap<(QueryId, Execution), Variable>, + read_by_query: BTreeMap<(QueryId, usize), Variable>, } /// $x$, $y$, $z$, $t$, $h$ and $r$ over `inputs`, added to `problem`: $x$ -/// fixed at 0 for an execution that could not be costed or is `left_out`, -/// and no $y$ on an edge of one. +/// fixed at 0 for an execution that is not `kept` (not costed, or dearer +/// than a feasible plan), and no $y$ on an edge of one. fn selection_variables( inputs: &SelectionInputs<'_>, - left_out: &[bool], + kept: &[bool], components: &Components, problem: &mut ProblemVariables, ) -> SelectionVariables { - let contexts = inputs.contexts; - let x = inputs - .costs - .per_execution + let selectable = inputs.selectable; + let x = kept .iter() - .zip(left_out) - .map(|(cost, &left_out)| match cost.is_ok() && !left_out { + .map(|&kept| match kept { true => problem.add(variable().binary()), false => problem.add(variable().min(0).max(0)), }) .collect(); - let y = contexts + let y = selectable .edges() .iter() .zip(&inputs.costs.per_edge) .enumerate() .filter(|(_, (&(consumer, _, producer), cost))| { - cost.is_some() - && !left_out[contexts.index_of(consumer)] - && !left_out[contexts.index_of(producer)] + cost.is_some() && kept[consumer] && kept[producer] }) .map(|(position, _)| (position, problem.add(variable().binary()))) .collect(); let mut z = BTreeMap::new(); - for (query, executions) in inputs.query_outputs { + for (query, executions) in &selectable.answering { for &execution in executions { z.entry((*query, execution)) .or_insert_with(|| problem.add(variable().binary())); @@ -128,7 +116,7 @@ fn selection_variables( .iter() .map(|&size| (size > 1).then(|| problem.add(variable().min(0).max((size - 1) as f64)))) .collect(); - let h = retention_variables(inputs, problem); + let h = retention_variables(inputs, kept, problem); let mut raw_data_on_disk = BTreeMap::new(); for source in inputs.costs.raw_data_source.values() { raw_data_on_disk @@ -153,23 +141,16 @@ fn input_satisfaction_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let physical = inputs.physical; - let contexts = inputs.contexts; - let mut producers: HashMap<(usize, usize), Expression> = HashMap::new(); - for (&position, &y) in &variables.y { - let (consumer, edge, _) = contexts.edges()[position]; - *producers - .entry(( - contexts.index_of(consumer), - physical.edge(edge).input_index_of_consumer, - )) - .or_default() += y; - } + let selectable = inputs.selectable; let mut constraints = Vec::new(); - for (index, execution) in contexts.executions().iter().enumerate() { - for i in 0..physical.input_count(execution.candidate) { - let chosen = producers.remove(&(index, i)).unwrap_or_default(); - constraints.push(constraint!(chosen == variables.x[index])); + for (index, &x) in variables.x.iter().enumerate() { + for i in 0..selectable.input_count(index) { + let chosen: Expression = selectable + .edges_into(index, i) + .iter() + .filter_map(|position| variables.y.get(position)) + .sum(); + constraints.push(constraint!(chosen == x)); } } constraints @@ -180,13 +161,12 @@ fn producer_selection_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { - let contexts = inputs.contexts; variables .y .iter() .map(|(&position, &y)| { - let (_, _, producer) = contexts.edges()[position]; - constraint!(y <= variables.x[contexts.index_of(producer)]) + let (_, _, producer) = inputs.selectable.edges()[position]; + constraint!(y <= variables.x[producer]) }) .collect() } @@ -198,7 +178,7 @@ fn query_coverage_constraints( variables: &SelectionVariables, ) -> Vec { let mut constraints = Vec::new(); - for (query, executions) in inputs.query_outputs { + for (query, executions) in &inputs.selectable.answering { let covered: Expression = executions .iter() .map(|&execution| variables.z[&(*query, execution)]) @@ -206,9 +186,7 @@ fn query_coverage_constraints( constraints.push(constraint!(covered == 1)); } for (&(_, execution), &z) in &variables.z { - constraints.push(constraint!( - z <= variables.x[inputs.contexts.index_of(execution)] - )); + constraints.push(constraint!(z <= variables.x[execution])); } constraints } @@ -221,13 +199,11 @@ fn acyclicity_constraints( variables: &SelectionVariables, components: &Components, ) -> Vec { - let contexts = inputs.contexts; variables .y .iter() .filter_map(|(&position, &y)| { - let (consumer, _, producer) = contexts.edges()[position]; - let (u, c) = (contexts.index_of(producer), contexts.index_of(consumer)); + let (c, _, u) = inputs.selectable.edges()[position]; if components.of[u] != components.of[c] { return None; } @@ -239,14 +215,13 @@ fn acyclicity_constraints( } /// (F) $x_{\kappa,v} \le \sum y + \sum_q z_{q,v}$ over the edges and query -/// outputs reading that execution; with it, a query-time result that is not -/// materialized ($\ell$ = None) has at most one consumer per context, -/// $\sum y + \sum_q z_{q,v} \le x_{\kappa,v}$, so for such an execution the -/// two are one row, $\sum y + \sum_q z_{q,v} = x_{\kappa,v}$. Consumers in -/// one context that share a result read a `OneEvaluation` or a materialized -/// producer (section 5). An ingestion-time producer is exempt from the -/// one-consumer rule: there is no `OneEvaluation` at ingestion time, and the -/// consumers of one ingestion run share its rows. +/// outputs reading that execution; with it, a candidate with one reader +/// per context ([`has_one_reader_per_context`]: a query-time result that +/// is not materialized) has $\sum y + \sum_q z_{q,v} \le x_{\kappa,v}$, so +/// for such an execution the two are one row, $\sum y + \sum_q z_{q,v} = +/// x_{\kappa,v}$. An ingestion-time producer is exempt from the +/// one-consumer rule: there is no `OneEvaluation` at ingestion time, and +/// the consumers of one ingestion run share its rows. fn use_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, @@ -255,11 +230,10 @@ fn use_constraints( .into_iter() .enumerate() .map(|(index, used)| { - let v = inputs.contexts.executions()[index].candidate; - let properties = inputs.physical.physical_candidates[v.0].physical_properties; - match properties.lifetime() == Lifetime::None - && properties.timing() == ExecutionTiming::QueryTime - { + let v = inputs.selectable.executions()[index].candidate; + match has_one_reader_per_context( + inputs.physical.physical_candidates[v.0].physical_properties, + ) { true => constraint!(used == variables.x[index]), false => constraint!(variables.x[index] <= used), } @@ -269,12 +243,13 @@ fn use_constraints( /// The raw data of each source is kept on disk once, for the selected scan /// that keeps the most: $r_s \ge \pi_v h_v$ for each query-time `Scan` $v$ -/// of source $s$ with $h_v$, and $r_s \ge d_v x_{\kappa,v}$ for each +/// of source $s$ with $h_v$, and $r_s \ge d_v x_{\kappa,v}$ for each kept /// execution of one without, $d_v$ its cost per second of the raw data it /// keeps retaining everything. fn raw_data_disk_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, + kept: &[bool], ) -> Vec { let costs = inputs.costs; let mut scans: Vec<(&PhysicalCandidateId, &Source)> = costs.raw_data_source.iter().collect(); @@ -283,11 +258,11 @@ fn raw_data_disk_constraints( for (v, source) in scans { let r = variables.raw_data_on_disk[source]; if let Some(&h) = variables.h.get(v) { - let per_second_retained = costs.retention_cost_per_second[v] * 1_000.0; - constraints.push(constraint!(r >= per_second_retained * h)); + let per_retained_second = costs.retention_cost_per_retained_second[v]; + constraints.push(constraint!(r >= per_retained_second * h)); } else if let Some(&d) = costs.unbounded_raw_data_on_disk_cost_per_second.get(v) { - for (index, execution) in inputs.contexts.executions().iter().enumerate() { - if execution.candidate == *v { + for (index, execution) in inputs.selectable.executions().iter().enumerate() { + if execution.candidate == *v && kept[index] { constraints.push(constraint!(r >= d * variables.x[index])); } } @@ -299,14 +274,13 @@ fn raw_data_disk_constraints( /// Per execution, its selected edges to consumers and query outputs: /// $\sum y + \sum_q z_{q,v}$. fn uses_of(inputs: &SelectionInputs<'_>, variables: &SelectionVariables) -> Vec { - let contexts = inputs.contexts; let mut uses: Vec = vec![Expression::default(); variables.x.len()]; for (&position, &y) in &variables.y { - let (_, _, producer) = contexts.edges()[position]; - uses[contexts.index_of(producer)] += y; + let (_, _, producer) = inputs.selectable.edges()[position]; + uses[producer] += y; } for (&(_, execution), &z) in &variables.z { - uses[contexts.index_of(execution)] += z; + uses[execution] += z; } uses } @@ -321,14 +295,15 @@ pub(crate) struct SolvedSelection { pub(crate) executions: Vec, /// The selected edges as (consumer, edge, producer), ascending. pub(crate) edges: Vec<(Execution, PhysicalEdgeId, Execution)>, - /// The execution answering each of `SelectionInputs::query_outputs`, in - /// its order. + /// The execution answering each of `SelectableExecutions::answering`, + /// in its order. pub(crate) outputs: Vec, /// $h_v$ in ms of each selected candidate with $h_v$: the least (R) /// allows for the selection. pub(crate) retention_ms: HashMap, - /// The optimum, unscaled: the selected plan's cost per second. - pub(crate) objective_cost_per_second: f64, + /// The optimum, unscaled: the selected plan's cost per second, less + /// the request overhead every plan pays. + pub(crate) cost_per_second_without_request_overhead: f64, /// The MILP's variables and constraint rows. pub(crate) variable_count: usize, pub(crate) constraint_count: usize, @@ -352,40 +327,43 @@ pub(crate) enum SelectionMilpError { /// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], /// [`raw_data_disk_constraints`], (R) and (X), with HiGHS at gap 0 on one /// thread. The executions dearer than a feasible plan are left out first -/// ([`dearer_than_a_feasible_plan`]), then the coefficients are scaled so -/// the largest left is 1,000 (HiGHS's tolerances are absolute). +/// ([`dearer_than_a_feasible_plan`]), with the $h_v$ of a candidate whose +/// executions are all left out, then the coefficients kept are scaled so +/// the largest is 1,000 (HiGHS's tolerances are absolute). pub(crate) fn solve_selection( inputs: &SelectionInputs<'_>, ) -> Result { let started = Instant::now(); - let left_out = dearer_than_a_feasible_plan(inputs); + let costs = inputs.costs; + let kept: Vec = dearer_than_a_feasible_plan(inputs) + .iter() + .zip(&costs.per_execution) + .map(|(&left_out, cost)| cost.is_ok() && !left_out) + .collect(); let components = Components::of(inputs); let mut problem = ProblemVariables::new(); - let variables = selection_variables(inputs, &left_out, &components, &mut problem); + let variables = selection_variables(inputs, &kept, &components, &mut problem); let constraints: Vec = [ input_satisfaction_constraints(inputs, &variables), producer_selection_constraints(inputs, &variables), query_coverage_constraints(inputs, &variables), acyclicity_constraints(inputs, &variables, &components), use_constraints(inputs, &variables), - raw_data_disk_constraints(inputs, &variables), + raw_data_disk_constraints(inputs, &variables, &kept), retention_constraints(inputs, &variables.h, &variables.y, &variables.z), exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), ] .into_iter() .flatten() .collect(); - let costs = inputs.costs; - // π_v per second of retention, the unit of h. - let retention_cost = |v: &PhysicalCandidateId| costs.retention_cost_per_second[v] * 1_000.0; // The objective's terms per second, unscaled: (cost, variable). A // query-time scan's retention is in its source's r_s. let terms: Vec<(f64, Variable)> = costs .per_execution .iter() .zip(&variables.x) - .zip(&left_out) - .filter(|(_, &left_out)| !left_out) + .zip(&kept) + .filter(|(_, &kept)| kept) .filter_map(|((cost, &x), _)| Some((*cost.as_ref().ok()?, x))) .chain( variables @@ -398,27 +376,35 @@ pub(crate) fn solve_selection( .h .iter() .filter(|(v, _)| !costs.raw_data_source.contains_key(v)) - .map(|(v, &h)| (retention_cost(v), h)), + .map(|(v, &h)| (costs.retention_cost_per_retained_second[v], h)), ) .chain(variables.raw_data_on_disk.values().map(|&r| (1.0, r))) .collect(); - // Scaled by the largest coefficient left, the retention and raw-data - // bounds included. + // Scaled by the largest coefficient kept, the raw-data bounds of the + // scans kept included. + let raw_data_bounds = + costs + .raw_data_source + .keys() + .filter_map(|v| match variables.h.contains_key(v) { + true => Some(costs.retention_cost_per_retained_second[v]), + false => costs + .unbounded_raw_data_on_disk_cost_per_second + .get(v) + .copied() + .filter(|_| { + inputs + .selectable + .executions() + .iter() + .zip(&kept) + .any(|(execution, &kept)| kept && execution.candidate == *v) + }), + }); let largest = terms .iter() .map(|(cost, _)| *cost) - .chain(costs.raw_data_source.keys().filter_map(|v| { - costs - .retention_cost_per_second - .get(v) - .map(|_| retention_cost(v)) - })) - .chain( - costs - .unbounded_raw_data_on_disk_cost_per_second - .values() - .copied(), - ) + .chain(raw_data_bounds) .fold(0.0_f64, |largest, cost| largest.max(cost.abs())); let scale = match largest > 0.0 && largest.is_finite() { true => 1_000.0 / largest, @@ -465,7 +451,8 @@ pub(crate) fn solve_selection( other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), }; let chosen = |variable: Variable| columns[column_of[&variable]] > 0.5; - let contexts = inputs.contexts; + let selectable = inputs.selectable; + let executions = selectable.executions(); let selected: Vec = (0..variables.x.len()) .filter(|&index| chosen(variables.x[index])) .collect(); @@ -475,38 +462,35 @@ pub(crate) fn solve_selection( .filter(|&(_, &y)| chosen(y)) .map(|(&position, _)| position) .collect(); - let outputs: Vec = inputs - .query_outputs + let answering: Vec<(QueryId, usize)> = selectable + .answering .iter() - .map(|(query, executions)| { - executions + .map(|(query, answering)| { + let execution = answering .iter() .copied() .find(|&execution| chosen(variables.z[&(*query, execution)])) - .expect("(C): one output per query output reference") + .expect("(C): one output per query output reference"); + (*query, execution) }) .collect(); - let answering: Vec<(QueryId, Execution)> = inputs - .query_outputs - .iter() - .zip(&outputs) - .map(|((query, _), &execution)| (*query, execution)) - .collect(); - let (cost_per_second, retention_ms) = + let (cost_per_second_without_request_overhead, retention_ms) = selection_cost_per_second(inputs, &selected, &selected_edges, &answering); - let objective_cost_per_second = cost_per_second + costs.request_overhead_cost_per_second; Ok(SolvedSelection { - executions: selected + executions: selected.iter().map(|&index| executions[index]).collect(), + edges: selected_edges .iter() - .map(|&index| contexts.executions()[index]) + .map(|&position| { + let (consumer, edge, producer) = selectable.edges()[position]; + (executions[consumer], edge, executions[producer]) + }) .collect(), - edges: selected_edges + outputs: answering .iter() - .map(|&position| contexts.edges()[position]) + .map(|&(_, execution)| executions[execution]) .collect(), - outputs, retention_ms, - objective_cost_per_second, + cost_per_second_without_request_overhead, variable_count, constraint_count, construction_ms, @@ -514,24 +498,25 @@ pub(crate) fn solve_selection( }) } -/// The cost per second of a selection, without the request overhead every +/// The cost per second of a selection, less the request overhead every /// plan pays: its executions (`selected`, by position in -/// `ExecutionContexts::executions`) and edges (`edges`, by position in -/// `ExecutionContexts::edges`), the retention they and the query outputs -/// `answering` need ([`least_retention_ms`]), and each source's raw data -/// on disk at its largest selected scan; with that retention. +/// `SelectableExecutions::executions`) and edges (`edges`, by position in +/// `SelectableExecutions::edges`), the retention they and the query +/// outputs `answering` (by execution position) need +/// ([`least_retention_ms`]), and each source's raw data on disk at its +/// largest selected scan; with that retention. pub(crate) fn selection_cost_per_second( inputs: &SelectionInputs<'_>, selected: &[usize], edges: &[usize], - answering: &[(QueryId, Execution)], + answering: &[(QueryId, usize)], ) -> (f64, HashMap) { let costs = inputs.costs; let retention_ms = least_retention_ms(inputs, edges.iter().copied(), answering.iter().copied()); let mut raw_data_on_disk: BTreeMap<&Source, f64> = BTreeMap::new(); let mut retention_cost_per_second = 0.0; for (&v, &ms) in &retention_ms { - let cost = costs.retention_cost_per_second[&v] * ms as f64; + let cost = costs.retention_cost_per_retained_second[&v] * ms as f64 / 1_000.0; match costs.raw_data_source.get(&v) { Some(source) => { let kept = raw_data_on_disk.entry(source).or_default(); @@ -541,7 +526,7 @@ pub(crate) fn selection_cost_per_second( } } for &index in selected { - let v = inputs.contexts.executions()[index].candidate; + let v = inputs.selectable.executions()[index].candidate; if let (Some(source), Some(&cost)) = ( costs.raw_data_source.get(&v), costs.unbounded_raw_data_on_disk_cost_per_second.get(&v), diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 25f66fb56..ce1ce3580 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -8,7 +8,7 @@ use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, Operator, QueryRoot}; +use asap_types::ir::{ASAPOp, Operator}; use asap_types::stages::{ CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, @@ -28,8 +28,10 @@ use asap_types::workload::{ use super::*; use crate::cost::ExecutionCosts; +use crate::milp::exact_only::must_not_read; use crate::plan::plan_of; -use crate::selectable::ExecutionContexts; +use crate::retention::edge_retention_need_ms; +use crate::selectable::output_groups; const NONE: Lifetime = Lifetime::None; const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); @@ -133,48 +135,38 @@ fn every(interval_ms: u32) -> QueryWorkloadEntry { } } -/// Each query output reference of `physical`'s memo: query `q` reads its -/// operator root. -fn outputs(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, MemoGroupId)> { - physical - .logical_candidates - .memo - .queries() - .iter() - .enumerate() - .map(|(query, memo_query)| match memo_query.root { - QueryRoot::Operator(group) => (QueryId(query), group), - QueryRoot::Scalar(_) => unreachable!("operator roots only"), - }) - .collect() -} - -/// Solves over `physical` for `entries` with every execution of candidate -/// `v` costing `w[v]`, every edge from `u` to `c` `edge_costs[(u, c)]` (0 -/// when missing) and candidate `v` retaining at `retention[v]` per ms; -/// returns the solution and the contexts it is over. -fn solve( +/// What the MILP is built over for `physical` and `entries`, with every +/// execution of candidate `v` costing `w[v]`, every edge from `u` to `c` +/// `edge_costs[(u, c)]` (0 when missing) and candidate `v` retaining at +/// `retention[v]` per second retained; each query output answered, as +/// `crate::select_plan` does, by the executions its query may read (X). +fn selection_inputs( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], w: &[f64], edge_costs: &[((usize, usize), f64)], retention: &[(usize, f64)], -) -> ( - Result, - ExecutionContexts, -) { - let (contexts, query_outputs) = ExecutionContexts::new(physical, entries, &outputs(physical)); +) -> (SelectableExecutions, ExecutionCosts) { + let mut selectable = SelectableExecutions::new(physical, entries, &output_groups(physical)); + selectable.retain_answering(|query, _, execution| { + !must_not_read(physical, query, execution.candidate) + }); + let retention_cost_per_retained_second: HashMap = retention + .iter() + .map(|&(v, per_retained_second)| (PhysicalCandidateId(v), per_retained_second)) + .collect(); + let edge_of = |&(_, id, _): &(usize, PhysicalEdgeId, usize)| physical.edge(id); let costs = ExecutionCosts { - per_execution: contexts + per_execution: selectable .executions() .iter() .map(|execution| Ok(w[execution.candidate.0])) .collect(), - per_edge: contexts + per_edge: selectable .edges() .iter() - .map(|&(_, id, _)| { - let edge = physical.edge(id); + .map(|selectable_edge| { + let edge = edge_of(selectable_edge); let key = (edge.producer_candidate_id.0, edge.consumer_candidate_id.0); Some( edge_costs @@ -184,22 +176,44 @@ fn solve( ) }) .collect(), - retention_cost_per_second: retention + retention_need_ms_per_edge: selectable + .edges() .iter() - .map(|&(v, per_ms)| (PhysicalCandidateId(v), per_ms)) + .map(|selectable_edge| { + let edge = edge_of(selectable_edge); + retention_cost_per_retained_second + .contains_key(&edge.producer_candidate_id) + .then(|| edge_retention_need_ms(physical, entries, selectable_edge.1)) + .flatten() + }) .collect(), + retention_cost_per_retained_second, raw_data_source: HashMap::new(), unbounded_raw_data_on_disk_cost_per_second: HashMap::new(), - request_overhead_cost_per_second: 0.0, }; + (selectable, costs) +} + +/// Solves over [`selection_inputs`]; returns the solution and the +/// executions it is over. +fn solve( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + w: &[f64], + edge_costs: &[((usize, usize), f64)], + retention: &[(usize, f64)], +) -> ( + Result, + SelectableExecutions, +) { + let (selectable, costs) = selection_inputs(physical, entries, w, edge_costs, retention); let solved = solve_selection(&SelectionInputs { physical, entries, - contexts: &contexts, + selectable: &selectable, costs: &costs, - query_outputs: &query_outputs, }); - (solved, contexts) + (solved, selectable) } /// The candidates of `solved`'s executions, one per execution, ascending. @@ -236,7 +250,7 @@ fn selects_the_producer_whose_own_and_edge_costs_are_lowest() { &every_10s(1), &[1.0, 2.0, 1.0], &[((0, 2), 5.0)], - &[(1, 1e-9)], + &[(1, 1e-6)], ); let solved = solved.unwrap(); assert_eq!(candidates(&solved), vec![1, 2]); @@ -244,7 +258,7 @@ fn selects_the_producer_whose_own_and_edge_costs_are_lowest() { assert_eq!(solved.edges[0].1, edge(&physical, 1, 2)); assert_eq!(solved.outputs[0].candidate, PhysicalCandidateId(2)); - let plan = plan_of(&physical, &solved, &outputs(&physical)).unwrap(); + let plan = plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); assert_eq!(plan.roots, vec![1]); assert_eq!(plan.edges.len(), 1); assert_eq!((plan.edges[0].producer, plan.edges[0].consumer), (0, 1)); @@ -342,7 +356,7 @@ fn a_result_that_is_not_materialized_runs_once_per_context_that_reads_it() { .map(|execution| execution.context) .collect(); assert_ne!(scan_contexts[0], scan_contexts[1]); - let plan = plan_of(&physical, &solved, &outputs(&physical)).unwrap(); + let plan = plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); assert_eq!(plan.nodes.len(), 4); assert_eq!(contexts.executions().len(), 6); } @@ -384,7 +398,7 @@ fn a_materialized_chain_runs_once_and_retains_what_its_consumers_read_back() { &[every(10_000), every(20_000)], &[1.0, 1.0, 1.0, 1.0], &[], - &[(1, 1e-6)], + &[(1, 1e-3)], ); let solved = solved.unwrap(); assert_eq!(candidates(&solved), vec![0, 1, 2, 3]); @@ -392,7 +406,7 @@ fn a_materialized_chain_runs_once_and_retains_what_its_consumers_read_back() { solved.retention_ms.get(&PhysicalCandidateId(1)), Some(&300_000) ); - let plan = plan_of(&physical, &solved, &outputs(&physical)).unwrap(); + let plan = plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); let chain_node = plan .nodes .iter() @@ -440,7 +454,7 @@ fn acyclicity_forbids_two_groups_reading_each_other() { selected == vec![0, 1, 4] || selected == vec![0, 2, 3], "{selected:?}" ); - plan_of(&physical, &solved, &outputs(&physical)).unwrap(); + plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); } /// x08's shape: `max by (job)(count_over_time(m[1m]))`. Path A (scan and @@ -491,6 +505,12 @@ fn a_candidate_dearer_than_a_feasible_plan_does_not_hide_a_cheaper_plan() { ]; let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[]); assert_eq!(candidates(&solved.unwrap()), vec![3, 4, 5, 6, 7, 8]); + // The unused candidate also retains rows at 2.55e6/s per second + // retained (z05's unused UnivMon chain): it has no h_v once its + // executions are left out, so its retention cost does not scale the + // objective either. + let (solved, _) = solve(&physical, &every_10s(1), &w, &[], &[(9, w[9])]); + assert_eq!(candidates(&solved.unwrap()), vec![3, 4, 5, 6, 7, 8]); } /// `quantile_over_time(0.9, m[5m])` read by Q0 every 10 s and Q1 every @@ -565,3 +585,84 @@ fn a_query_reads_its_exact_only_groups_through_exact_candidates_only() { ); } } + +/// x07's shape: `quantile_over_time(0.9, m[5m])` read by Q0 every 10 s, +/// whose only paths read the KLL state (its exact quantile cannot be +/// costed), and by Q1 every 30 s, which may read every group only exactly. +/// The greedy bound leaves out of each query's trees only what that query +/// must not read, so it finds a plan (Q0 through the state, Q1 through the +/// exact quantile) and leaves out an unused scan costing 1e6/s; leaving +/// out of every query's trees what some query must not read found none. +#[test] +fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let window = range(&mut memo, 300, m); + let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); + let state = quantile.by_algorithm[0].state; + query(&mut memo, quantile.result, None); + query(&mut memo, quantile.result, None); + let mut candidates: Vec<(CandidateForMemoGroupId, Lifetime)> = (0..memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .flat_map(|group| { + let count = match group == state { + true => 1, + false => memo.groups()[group.0 as usize].candidates.len(), + }; + (0..count).map(move |index_in_group| { + ( + CandidateForMemoGroupId { + group, + index_in_group, + }, + ONE_EVALUATION, + ) + }) + }) + .collect(); + let exact_quantile = candidates + .iter() + .position(|&(candidate, _)| candidate == own_candidate(quantile.result)) + .unwrap(); + let unused_scan = candidates.len(); + candidates.push((own_candidate(m), NONE)); + let w: Vec = (0..candidates.len()) + .map(|v| match v { + _ if v == unused_scan => 1e6, + _ if v == exact_quantile => 100.0, + _ => 1.0, + }) + .collect(); + let mut physical = physical(memo, &candidates); + physical.logical_candidates.exact_only_groups_of_query = vec![ + BTreeSet::new(), + (0..physical.logical_candidates.memo.groups().len()) + .map(|group| MemoGroupId(group as u32)) + .collect(), + ]; + let entries = [every(10_000), every(30_000)]; + let (mut selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let q0_context = selectable.executions()[selectable.answering[0].1[0]].context; + for (index, execution) in selectable.executions().iter().enumerate() { + if execution.candidate.0 == exact_quantile && execution.context == q0_context { + costs.per_execution[index] = Err("not costable".into()); + } + } + selectable.retain_answering(|_, index, _| costs.per_execution[index].is_ok()); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let left_out = dearer_than_a_feasible_plan(&inputs); + let unused: Vec = selectable + .executions() + .iter() + .zip(&left_out) + .filter(|(execution, _)| execution.candidate.0 == unused_scan) + .map(|(_, &left_out)| left_out) + .collect(); + assert_eq!(unused, vec![true, true]); + assert!(solve_selection(&inputs).is_ok()); +} diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs new file mode 100644 index 000000000..a7cd0b41d --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -0,0 +1,117 @@ +//! The retention extension of the selection MILP +//! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, +//! section 5): the retention amount $h_v$ and constraint (R). The +//! executions $x_{\kappa,v}$ and edges $y_{\kappa,u,c,i}$ it selects over +//! are [`crate::selectable::SelectableExecutions`]'s, in [`super::model`]. + +use std::collections::{BTreeMap, HashMap}; + +use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; + +use asap_types::stages::PhysicalCandidateId; + +use super::model::SelectionInputs; +use crate::retention::output_retention_need_ms; +use crate::selectable::QueryId; + +/// $h_v$ in seconds retained (the unit keeps (R)'s coefficients near the +/// others'), added to `problem`, for each candidate with a retention cost +/// ([`crate::cost::ExecutionCosts::retention_cost_per_retained_second`]) +/// one of whose executions is `kept` (by position in +/// `SelectableExecutions::executions`): one whose executions are all left +/// out retains nothing. +pub(crate) fn retention_variables( + inputs: &SelectionInputs<'_>, + kept: &[bool], + problem: &mut ProblemVariables, +) -> BTreeMap { + let retaining = &inputs.costs.retention_cost_per_retained_second; + let mut retained: Vec = inputs + .selectable + .executions() + .iter() + .zip(kept) + .filter(|(execution, &kept)| kept && retaining.contains_key(&execution.candidate)) + .map(|(execution, _)| execution.candidate) + .collect(); + retained.sort_unstable(); + retained.dedup(); + retained + .into_iter() + .map(|v| (v, problem.add(variable().min(0)))) + .collect() +} + +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$ and +/// $h_v \ge \operatorname{span}_q\, z_{q,v}$, for a candidate $v$ with $h_v$: +/// how far back its rows are read through each selectable edge `y` (the +/// reading path's offset plus range, +/// [`crate::cost::ExecutionCosts::retention_need_ms_per_edge`]) and by each +/// query reading it directly (its range-evaluation span), a constant per +/// edge, so (R) is linear. Query latency is ignored: a result counts as +/// read when its evaluation starts. +pub(crate) fn retention_constraints( + inputs: &SelectionInputs<'_>, + h: &BTreeMap, + y: &BTreeMap, + z: &BTreeMap<(QueryId, usize), Variable>, +) -> Vec { + let executions = inputs.selectable.executions(); + let mut constraints = Vec::new(); + for (&position, &y) in y { + let (_, _, producer) = inputs.selectable.edges()[position]; + let Some(&h) = h.get(&executions[producer].candidate) else { + continue; + }; + let need_ms = inputs.costs.retention_need_ms_per_edge[position] + .expect("a candidate with h_v is read within a bound"); + constraints.push(constraint!(h >= seconds(need_ms) * y)); + } + for (&(query, execution), &z) in z { + if let Some(&h) = h.get(&executions[execution].candidate) { + let need_ms = output_retention_need_ms(&inputs.entries[query.0]); + constraints.push(constraint!(h >= seconds(need_ms) * z)); + } + } + constraints +} + +/// The least $h_v$ (R) allows for a selection, in ms, of each candidate +/// with a retention cost that `edges` (selected, by position in +/// `SelectableExecutions::edges`) or `outputs` (the query outputs and the +/// executions answering them, by position) read: what the MILP's optimum +/// retains when $\pi_v > 0$, and what the plan retains. +pub(crate) fn least_retention_ms( + inputs: &SelectionInputs<'_>, + edges: impl IntoIterator, + outputs: impl IntoIterator, +) -> HashMap { + let executions = inputs.selectable.executions(); + let retaining = &inputs.costs.retention_cost_per_retained_second; + let reads = edges + .into_iter() + .filter_map(|position| { + let (_, _, producer) = inputs.selectable.edges()[position]; + Some(( + executions[producer].candidate, + inputs.costs.retention_need_ms_per_edge[position]?, + )) + }) + .chain(outputs.into_iter().map(|(query, execution)| { + ( + executions[execution].candidate, + output_retention_need_ms(&inputs.entries[query.0]), + ) + })); + let mut retention_ms: HashMap = HashMap::new(); + for (v, need_ms) in reads.filter(|(v, _)| retaining.contains_key(v)) { + let most = retention_ms.entry(v).or_default(); + *most = (*most).max(need_ms); + } + retention_ms +} + +/// `ms` in seconds, the unit of $h_v$ in the MILP. +fn seconds(ms: u64) -> f64 { + ms as f64 / 1_000.0 +} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index ac29dbddb..cf21eea51 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -12,147 +12,112 @@ //! greedy plan bounds the optimum by about 0.0088/s, so the chain is left //! out and the largest coefficient left is about 0.006/s. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::Lifetime; - -use super::exact_only::not_read_by_some_query; -use super::model::{selection_cost_per_second, QueryId, SelectionInputs}; -use crate::selectable::Execution; +use super::exact_only::must_not_read; +use super::model::{selection_cost_per_second, SelectionInputs}; +use crate::selectable::{has_one_reader_per_context, QueryId}; /// How many times the greedy plan is rebuilt without an execution that /// made it infeasible before giving up on a bound. const REPAIRS: usize = 32; -/// Per (consumer execution, input), the costed edges into it, each as -/// (edge position, producer execution, $\rho_\kappa c_{\kappa,u,c,i}$). -type EdgesInto = HashMap<(usize, usize), Vec<(usize, usize, f64)>>; - /// Per input of an execution, the edge its cheapest tree reads it over, as /// (edge position, producer execution). type TreeInputs = Vec>; -/// Per execution, by position in `ExecutionContexts::executions`, whether -/// it costs more than a feasible plan ([`feasible_plan_cost_per_second`]): -/// its own cost plus the most its input edges can take off ($\min(0, -/// c_{\kappa,u,v,i})$ per input) is over the bound. Such an execution is in -/// no optimal plan, as long as every other selected execution costs at -/// least 0 with its selected inputs. None is ruled out when no feasible -/// plan is found. +/// Per execution, by position in `SelectableExecutions::executions`, +/// whether it costs more than a feasible plan +/// ([`feasible_plan_cost_per_second`]): its own cost plus the most its +/// input edges can take off ($\min(0, c_{\kappa,u,v,i})$ per input) is over +/// the bound. Such an execution is in no optimal plan, as long as every +/// other selected execution costs at least 0 with its selected inputs. +/// None is ruled out when no feasible plan is found. pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec { - let contexts = inputs.contexts; - let count = contexts.executions().len(); + let selectable = inputs.selectable; + let count = selectable.executions().len(); let Some(bound) = feasible_plan_cost_per_second(inputs) else { return vec![false; count]; }; - let mut cheapest_input_edge: HashMap<(usize, usize), f64> = HashMap::new(); - for (&(consumer, edge, _), cost) in contexts.edges().iter().zip(&inputs.costs.per_edge) { - let Some(cost) = cost else { - continue; - }; - let key = ( - contexts.index_of(consumer), - inputs.physical.edge(edge).input_index_of_consumer, - ); - let cheapest = cheapest_input_edge.entry(key).or_insert(0.0); - *cheapest = cheapest.min(*cost); - } let tolerance = 1e-9 * bound.abs().max(1e-12); (0..count) .map(|index| { let Ok(own) = &inputs.costs.per_execution[index] else { return false; }; - let v = contexts.executions()[index].candidate; let least = own - + (0..inputs.physical.input_count(v)) - .filter_map(|i| cheapest_input_edge.get(&(index, i))) + + (0..selectable.input_count(index)) + .map(|i| { + selectable + .edges_into(index, i) + .iter() + .filter_map(|&position| inputs.costs.per_edge[position]) + .fold(0.0, f64::min) + }) .sum::(); least > bound + tolerance }) .collect() } -/// The cost per second (without the request overhead every plan pays) of -/// a feasible plan ([`selection_cost_per_second`]): each query output -/// reads the cheapest tree of executions below it, each execution's input -/// from the producer whose tree is cheapest ($\rho_\kappa c_{\kappa,v} + -/// \sum_i \min_u (\rho_\kappa c_{\kappa,u,v,i} + \text{tree}_u)$). Trees -/// share the executions they have in common; when that gives a query-time -/// result that is not materialized two consumers, or a cycle, the plan is -/// rebuilt without that execution. An execution some query must not read -/// (constraint (X)) is left out from the start, so the plan meets (X). -/// `None` when no feasible plan is found that way. +/// The cost per second (less the request overhead every plan pays) of a +/// feasible plan ([`selection_cost_per_second`]): each query output reads +/// the cheapest tree of executions below it, each execution's input from +/// the producer whose tree is cheapest ($\rho_\kappa c_{\kappa,v} + \sum_i +/// \min_u (\rho_\kappa c_{\kappa,u,v,i} + \text{tree}_u)$), over the +/// executions its query may read (constraint (X)). The query outputs are +/// added in order, a later tree reusing the executions an earlier one +/// added. When that gives an execution with one reader per context a +/// second consumer, closes a cycle, or reuses a tree with an execution +/// the query must not read, the plan is rebuilt with that execution left +/// out of the query's trees. `None` when no feasible plan is found that +/// way. fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { - let physical = inputs.physical; - let contexts = inputs.contexts; - let count = contexts.executions().len(); - let mut into: EdgesInto = HashMap::new(); - for (position, (&(consumer, edge, producer), cost)) in contexts - .edges() + let selectable = inputs.selectable; + let executions = selectable.executions(); + let count = executions.len(); + let one_reader: Vec = executions .iter() - .zip(&inputs.costs.per_edge) - .enumerate() - { - let Some(cost) = cost else { - continue; - }; - into.entry(( - contexts.index_of(consumer), - physical.edge(edge).input_index_of_consumer, - )) - .or_default() - .push((position, contexts.index_of(producer), *cost)); - } - let inputs_of: Vec = contexts - .executions() - .iter() - .map(|execution| physical.input_count(execution.candidate)) + .map(|execution| { + has_one_reader_per_context( + inputs.physical.physical_candidates[execution.candidate.0].physical_properties, + ) + }) .collect(); - let unshared = |index: usize| { - let v = contexts.executions()[index].candidate; - let properties = physical.physical_candidates[v.0].physical_properties; - properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime - }; - let mut excluded = not_read_by_some_query(inputs); - for _ in 0..REPAIRS { - let (tree_cost, chosen) = cheapest_trees(inputs, &into, &inputs_of, &excluded); - // Each query output's cheapest execution, then the trees below. - let mut uses = vec![0usize; count]; - let mut outputs = Vec::new(); - let mut answering: Vec<(QueryId, Execution)> = Vec::new(); - for (query, executions) in inputs.query_outputs { - let best = executions + let forbidden = + |query: QueryId, e: usize| must_not_read(inputs.physical, query, executions[e].candidate); + // Per query, the executions its trees leave out: those it must not + // read, and those a repair left out for it. + let mut excluded: BTreeMap> = BTreeMap::new(); + for (query, _) in &selectable.answering { + excluded + .entry(*query) + .or_insert_with(|| (0..count).map(|e| forbidden(*query, e)).collect()); + } + // The cheapest trees, per set of executions left out. + let mut trees: HashMap, (Vec, Vec)> = HashMap::new(); + 'repair: for _ in 0..REPAIRS { + let mut plan = GreedyPlan::new(count); + let mut answering: Vec<(QueryId, usize)> = Vec::new(); + for (query, outputs) in &selectable.answering { + let left_out = &excluded[query]; + let (tree_cost, chosen) = trees + .entry(left_out.clone()) + .or_insert_with(|| cheapest_trees(inputs, left_out)); + let best = outputs .iter() - .map(|&execution| contexts.index_of(execution)) + .copied() .filter(|&index| tree_cost[index].is_finite()) .min_by(|&a, &b| tree_cost[a].total_cmp(&tree_cost[b]))?; - uses[best] += 1; - outputs.push(best); - answering.push((*query, contexts.executions()[best])); - } - let plan = match plan_below(&outputs, &chosen, &inputs_of) { - Ok(plan) => plan, - Err(on_cycle) => { - excluded[on_cycle] = true; - continue; + answering.push((*query, best)); + if let Err(repair) = plan.add_tree(best, chosen, &one_reader, |e| forbidden(*query, e)) + { + excluded.get_mut(query).expect("a query")[repair] = true; + continue 'repair; } - }; - for &(_, producer) in &plan.edges { - uses[producer] += 1; - } - if let Some(shared) = plan - .executions - .iter() - .copied() - .find(|&index| unshared(index) && uses[index] > 1) - { - excluded[shared] = true; - continue; } - let edges: Vec = plan.edges.iter().map(|&(position, _)| position).collect(); - let (cost, _) = selection_cost_per_second(inputs, &plan.executions, &edges, &answering); + let (cost, _) = + selection_cost_per_second(inputs, &plan.executions, &plan.edges, &answering); return cost.is_finite().then_some(cost); } None @@ -162,15 +127,13 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { /// input, the edge it reads it over: relaxed until no tree gets cheaper /// (at most once per execution), so a tree over a cycle of executions is /// never cheaper than one without it when costs are not negative. -fn cheapest_trees( - inputs: &SelectionInputs<'_>, - into: &EdgesInto, - inputs_of: &[usize], - excluded: &[bool], -) -> (Vec, Vec) { - let count = inputs_of.len(); +fn cheapest_trees(inputs: &SelectionInputs<'_>, excluded: &[bool]) -> (Vec, Vec) { + let selectable = inputs.selectable; + let count = excluded.len(); let mut tree_cost = vec![f64::INFINITY; count]; - let mut chosen: Vec = inputs_of.iter().map(|&n| vec![None; n]).collect(); + let mut chosen: Vec = (0..count) + .map(|v| vec![None; selectable.input_count(v)]) + .collect(); for _ in 0..count.max(1) { let mut changed = false; for v in 0..count { @@ -181,14 +144,17 @@ fn cheapest_trees( continue; } let mut cost = *own; - let mut reads = Vec::with_capacity(inputs_of[v]); - for i in 0..inputs_of[v] { - let cheapest = into - .get(&(v, i)) - .into_iter() - .flatten() - .filter(|&&(_, u, _)| !excluded[u] && tree_cost[u].is_finite()) - .map(|&(position, u, c)| (position, u, c + tree_cost[u])) + let mut reads = Vec::with_capacity(selectable.input_count(v)); + for i in 0..selectable.input_count(v) { + let cheapest = selectable + .edges_into(v, i) + .iter() + .filter_map(|&position| { + let c = inputs.costs.per_edge[position]?; + let (_, _, u) = selectable.edges()[position]; + (!excluded[u] && tree_cost[u].is_finite()) + .then(|| (position, u, c + tree_cost[u])) + }) .min_by(|a, b| a.2.total_cmp(&b.2)); match cheapest { Some((position, u, through)) => { @@ -216,49 +182,93 @@ fn cheapest_trees( /// A plan of the trees below some query outputs. struct GreedyPlan { - /// Its executions, each once. + /// Its executions, each once, producers first. executions: Vec, - /// Its edges, as (edge position, producer execution). - edges: Vec<(usize, usize)>, + /// Its edges, by position in `SelectableExecutions::edges`. + edges: Vec, + /// Per execution, its producers in the plan; `None` when not in it yet. + producers: Vec>>, } -/// The plan of the trees `chosen` below `outputs`; Err with an execution -/// on a cycle of them. -fn plan_below( - outputs: &[usize], - chosen: &[TreeInputs], - inputs_of: &[usize], -) -> Result { - // 0 unvisited, 1 on the walk, 2 done. - let mut state = vec![0u8; inputs_of.len()]; - let mut plan = GreedyPlan { - executions: Vec::new(), - edges: Vec::new(), - }; - for &output in outputs { - if state[output] == 2 { - continue; +impl GreedyPlan { + /// The empty plan over `count` executions. + fn new(count: usize) -> Self { + Self { + executions: Vec::new(), + edges: Vec::new(), + producers: vec![None; count], + } + } + + /// Adds the tree `chosen` below `output` (each execution's inputs, by + /// execution), reusing the executions already in the plan with their + /// trees; Err with an execution to leave out of the query's trees: one + /// with `one_reader` per context read a second time, one closing a + /// cycle, or one already in the plan whose tree reads an execution + /// `forbidden` to the query. + fn add_tree( + &mut self, + output: usize, + chosen: &[TreeInputs], + one_reader: &[bool], + forbidden: impl Fn(usize) -> bool, + ) -> Result<(), usize> { + if self.producers[output].is_some() { + return self.reuse(output, one_reader, &forbidden); } + // Executions on the walk, not yet in the plan. + let mut on_walk = vec![false; self.producers.len()]; + on_walk[output] = true; let mut stack = vec![(output, 0usize)]; - state[output] = 1; + let mut walked: Vec> = vec![Vec::new(); self.producers.len()]; while let Some((v, i)) = stack.pop() { - if i == inputs_of[v] { - state[v] = 2; - plan.executions.push(v); + if i == chosen[v].len() { + on_walk[v] = false; + self.producers[v] = Some(std::mem::take(&mut walked[v])); + self.executions.push(v); continue; } stack.push((v, i + 1)); let (position, u) = chosen[v][i].expect("a finite tree reads every input"); - plan.edges.push((position, u)); - match state[u] { - 0 => { - state[u] = 1; + self.edges.push(position); + walked[v].push(u); + if on_walk[u] { + return Err(u); + } + match self.producers[u] { + Some(_) => self.reuse(u, one_reader, &forbidden)?, + None => { + on_walk[u] = true; stack.push((u, 0)); } - 1 => return Err(u), - _ => {} } } + Ok(()) + } + + /// Reads `execution`, already in the plan, a second time: Err with it + /// when it has `one_reader` per context or its tree reads an execution + /// `forbidden` to the reader's query. + fn reuse( + &self, + execution: usize, + one_reader: &[bool], + forbidden: &impl Fn(usize) -> bool, + ) -> Result<(), usize> { + if one_reader[execution] { + return Err(execution); + } + let mut pending = vec![execution]; + let mut seen = vec![false; self.producers.len()]; + while let Some(e) = pending.pop() { + if std::mem::replace(&mut seen[e], true) { + continue; + } + if forbidden(e) { + return Err(execution); + } + pending.extend(self.producers[e].iter().flatten()); + } + Ok(()) } - Ok(plan) } diff --git a/crates/stage3-plan-selection/src/plan.rs b/crates/stage3-plan-selection/src/plan.rs index 36f2552f1..e25fc8f0f 100644 --- a/crates/stage3-plan-selection/src/plan.rs +++ b/crates/stage3-plan-selection/src/plan.rs @@ -14,10 +14,10 @@ use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, }; use asap_types::plan::view_coverage; -use asap_types::stages::{Lifetime, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId}; +use asap_types::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId}; -use crate::milp::model::{QueryId, SolvedSelection}; -use crate::selectable::Execution; +use crate::milp::model::SolvedSelection; +use crate::selectable::{Execution, QueryId}; /// Candidate `v` as a plan node with id `id`, its input `i` read from node /// `input_ids[i]`: its operator, timing, lifetime and θ, its group's schema @@ -51,7 +51,9 @@ pub(crate) fn candidate_node( output_schema: group.schema.clone(), retention_ms: None, selected_physical_candidate: Some(v), - view_coverage: matches!(lifetime, Lifetime::AcrossEvaluations(_)) + view_coverage: candidate + .physical_properties + .is_across_evaluations() .then(|| view_coverage(memo, candidate.logical_candidate_id.group)) .flatten(), } diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index f399aaaea..35e0d5332 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -15,9 +15,10 @@ use asap_types::stages::{ use asap_types::workload::QueryWorkloadEntry; /// $\operatorname{need}_{\kappa,u,c,i}$: how far back from the evaluation -/// time `edge`'s producer $u$ must keep its rows for its consumer $c$ -/// ([`retention_need_ms`] with $c$ $u$'s only reader, and every candidate -/// that may read $c$ above it); `None` when read unbounded. +/// time `edge`'s producer $u$ must keep its rows for its consumer $c$, with +/// $c$ $u$'s only reader and every candidate that may read $c$ above it +/// ([`read_back`]), at each reader the longest range-evaluation span of +/// the queries reaching it; `None` when read unbounded. /// /// Example: `TimeRange(5m)` over a chain of 10 s windows, read by a range /// query with a 1 h span: 3,900,000 ms. @@ -26,19 +27,27 @@ pub(crate) fn edge_retention_need_ms( entries: &[QueryWorkloadEntry], edge: PhysicalEdgeId, ) -> Option { - let producer = physical.edge(edge).producer_candidate_id; - retention_need_ms(physical, entries, producer, readers_through(physical, edge)) + let memo = &physical.logical_candidates.memo; + read_back(physical, edge, |v| { + let group = physical.physical_candidates[v.0].logical_candidate_id.group; + memo.groups()[group.0 as usize] + .reached_by_queries() + .iter() + .filter_map(|&query| entries.get(query)?.range_evaluation) + .map(|range| range.span_ms) + .max() + .unwrap_or(0) + }) } /// How far back one execution of the query-time `Scan` that is `edge`'s -/// producer reads for its consumer ([`scan_read_ms`] with that consumer -/// its only reader); `None` when read whole. +/// producer reads for its consumer ([`read_back`] without a +/// range-evaluation span); `None` when read whole. pub(crate) fn edge_scan_read_ms( physical: &PhysicalASAPMemoCandidates, edge: PhysicalEdgeId, ) -> Option { - let producer = physical.edge(edge).producer_candidate_id; - scan_read_ms(physical, producer, readers_through(physical, edge)) + read_back(physical, edge, |_| 0) } /// How far back a query reads the output it reads directly: its @@ -47,67 +56,25 @@ pub(crate) fn output_retention_need_ms(entry: &QueryWorkloadEntry) -> u64 { entry.range_evaluation.map_or(0, |range| range.span_ms) } -/// The readers of each candidate when `edge`'s producer is read over -/// `edge` only: its consumer, and above it every candidate that may read -/// one (its admissible edges out). -fn readers_through( +/// [`asap_cost::read_back_ms`] of `edge`'s producer over candidates, read +/// by `edge`'s consumer only and above it by every candidate that may read +/// one (its admissible edges out), each read as its operator at its timing +/// reads ([`Candidates`]), with `span_ms` at each reader. +fn read_back( physical: &PhysicalASAPMemoCandidates, edge: PhysicalEdgeId, -) -> impl Fn(PhysicalCandidateId) -> Vec + '_ { + span_ms: impl Fn(PhysicalCandidateId) -> u64, +) -> Option { let edge = physical.edge(edge); - move |v| match v == edge.producer_candidate_id { + let producer = edge.producer_candidate_id; + let readers_of = |v: PhysicalCandidateId| match v == producer { true => vec![edge.consumer_candidate_id], false => physical .admissible_edges_out_of(v) .iter() .map(|&out| physical.edge(out).consumer_candidate_id) .collect(), - } -} - -/// How far back from the evaluation time `producer`'s rows must be kept -/// for its readers (`readers_of`, the candidates that may read it): -/// [`asap_cost::read_back_ms`] with, at each reader, the longest -/// range-evaluation span of the queries reaching it. -fn retention_need_ms>( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - producer: PhysicalCandidateId, - readers_of: impl Fn(PhysicalCandidateId) -> R, -) -> Option { - let memo = &physical.logical_candidates.memo; - let span_ms = |v: PhysicalCandidateId| { - let group = physical.physical_candidates[v.0].logical_candidate_id.group; - memo.groups()[group.0 as usize] - .reached_by_queries() - .iter() - .filter_map(|&query| entries.get(query)?.range_evaluation) - .map(|range| range.span_ms) - .max() - .unwrap_or(0) }; - read_back(physical, producer, readers_of, span_ms) -} - -/// How far back from the evaluation time one execution of the query-time -/// `Scan` `producer` reads, for its readers (`readers_of`): -/// [`asap_cost::read_back_ms`] without a range-evaluation span. -fn scan_read_ms>( - physical: &PhysicalASAPMemoCandidates, - producer: PhysicalCandidateId, - readers_of: impl Fn(PhysicalCandidateId) -> R, -) -> Option { - read_back(physical, producer, readers_of, |_| 0) -} - -/// [`asap_cost::read_back_ms`] over candidates ([`Candidates`]). -fn read_back>( - physical: &PhysicalASAPMemoCandidates, - producer: PhysicalCandidateId, - readers_of: impl Fn(PhysicalCandidateId) -> R, - span_ms: impl Fn(PhysicalCandidateId) -> u64, -) -> Option { - let readers_of = |v| readers_of(v).into_iter().collect(); let candidates = Candidates { physical, readers_of: &readers_of, diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index b8158261d..7dc151446 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -3,7 +3,7 @@ //! section 5, "Execution contexts, by example"): the distinct execution //! contexts $\kappa$ of a workload, every execution of a candidate in a //! context, the edges $y_{\kappa,u,c,i}$ between executions, and per query -//! output the executions that may answer it ([`ExecutionContexts::new`]). +//! output the executions that may answer it ([`SelectableExecutions::new`]). //! //! - A query is evaluated in the evaluation context of its grid //! ([`evaluation_grid`]); queries on one grid share it, a range query's @@ -19,14 +19,17 @@ //! producers must deliver on the ingestion pipeline of its refresh //! window. //! - The input of a `TimeShift` anchored with `@ end()` or `@ start()` runs -//! on the grid of the request end (start) times: once per request of a -//! range query, as an instant query with its recurrence is evaluated. +//! on the grid of the request end (start) times, once per request of a +//! range query, in a context of its own per anchor: a request's start, +//! its end and an evaluation are different times, so their results are +//! not shared. //! - An input of a consumer in $\kappa$ reads a producer, over an admissible //! edge, only where the producer delivers on $\kappa$ //! (`PhysicalCandidate::delivers_on`). use std::collections::BTreeSet; +use asap_cost::plan::execution_demand::{evaluation_grid_ms, request_time_grid_ms}; use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; @@ -34,11 +37,13 @@ use asap_types::plan::materialized_view::RefreshWindow; use asap_types::stages::{ output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, ExecutionContextId, Lifetime, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, - PhysicalEdgeId, + PhysicalEdgeId, PhysicalProperties, }; use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; -use crate::milp::model::QueryId; +/// A query, by index in the workload. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct QueryId(pub usize); /// The one context of every ingestion-time execution that is not /// materialized: each second of arriving rows, which it computes once for @@ -59,18 +64,34 @@ pub(crate) struct Execution { } /// The distinct contexts of a workload and every execution and edge the -/// selection may choose ([`Self::new`]). -pub(crate) struct ExecutionContexts { +/// selection may choose, with the executions that may answer each query +/// output ([`Self::new`]). Executions are referred to by position in +/// [`Self::executions`]. +pub(crate) struct SelectableExecutions { /// The distinct contexts, by [`ExecutionContextId`]. contexts: Vec, + /// Per context, by [`ExecutionContextId`]: for the inputs of a + /// `TimeShift` anchored with `@ end()` or `@ start()`, the anchor and + /// the evaluation grid of the queries whose requests they run for; + /// `None` for any other context. + request_anchor_of_context: Vec>, /// Every execution reached from a query output, ascending. executions: Vec, /// Every $y_{\kappa,u,c,i}$ as (consumer execution, edge $(u,c,i)$, - /// producer execution), ascending; $\kappa$ is the consumer's context. - edges: Vec<(Execution, PhysicalEdgeId, Execution)>, + /// producer execution), executions by position, ascending; $\kappa$ is + /// the consumer's context. + edges: Vec<(usize, PhysicalEdgeId, usize)>, + /// Per execution, by position, and per input of its candidate, the + /// positions in [`Self::edges`] of the edges into it. + edges_into: Vec>>, + /// Each query output reference (a query, and the group its operator + /// root, or one operator of its scalar root, reads), in + /// [`output_groups`] order, with the executions, by position, that may + /// answer it: $z_{q,v}$ exists only for them. + pub(crate) answering: Vec<(QueryId, Vec)>, } -impl ExecutionContexts { +impl SelectableExecutions { /// The contexts, executions and edges of `physical` for the queries /// `entries` (by query index), reached from `outputs` (each query output /// reference, a query and the group it reads, [`output_groups`]), and @@ -82,20 +103,26 @@ impl ExecutionContexts { physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], outputs: &[(QueryId, MemoGroupId)], - ) -> (Self, Vec<(QueryId, Vec)>) { - let mut contexts = Self { + ) -> Self { + let mut selectable = Self { contexts: Vec::new(), + request_anchor_of_context: Vec::new(), executions: Vec::new(), edges: Vec::new(), + edges_into: Vec::new(), + answering: Vec::new(), }; let mut reached: BTreeSet = BTreeSet::new(); let mut pending: Vec = Vec::new(); - let mut output_executions = Vec::with_capacity(outputs.len()); + let mut answering = Vec::with_capacity(outputs.len()); for &(query, group) in outputs { - let context = contexts.context_id(ExecutionContext::Evaluation { - grid: evaluation_grid(&entries[query.0]), - }); - let mut answering = Vec::new(); + let context = selectable.context_id( + ExecutionContext::Evaluation { + grid: evaluation_grid(&entries[query.0]), + }, + None, + ); + let mut executions = Vec::new(); for (v, candidate) in physical.physical_candidates.iter().enumerate() { if candidate.logical_candidate_id.group != group || !producer_satisfies_physical_requirements( @@ -106,24 +133,24 @@ impl ExecutionContexts { continue; } if let Some(execution) = - contexts.execution_reading(physical, context, PhysicalCandidateId(v)) + selectable.execution_reading(physical, context, PhysicalCandidateId(v)) { - answering.push(execution); + executions.push(execution); if reached.insert(execution) { pending.push(execution); } } } - output_executions.push((query, answering)); + answering.push((query, executions)); } let mut edges = BTreeSet::new(); while let Some(consumer) = pending.pop() { let c = consumer.candidate; - let context = contexts.input_context(physical, entries, consumer); + let context = selectable.input_context(physical, entries, consumer); for input in 0..physical.input_count(c) { for &edge in physical.admissible_edges_into(c, input) { let u = physical.edge(edge).producer_candidate_id; - if let Some(producer) = contexts.execution_reading(physical, context, u) { + if let Some(producer) = selectable.execution_reading(physical, context, u) { edges.insert((consumer, edge, producer)); if reached.insert(producer) { pending.push(producer); @@ -132,17 +159,50 @@ impl ExecutionContexts { } } } - contexts.executions = reached.into_iter().collect(); - contexts.edges = edges.into_iter().collect(); - (contexts, output_executions) + selectable.executions = reached.into_iter().collect(); + let position = |execution: Execution| { + selectable + .executions + .binary_search(&execution) + .expect("a reached execution") + }; + selectable.edges = edges + .into_iter() + .map(|(consumer, edge, producer)| (position(consumer), edge, position(producer))) + .collect(); + selectable.answering = answering + .into_iter() + .map(|(query, executions)| (query, executions.into_iter().map(position).collect())) + .collect(); + selectable.edges_into = selectable + .executions + .iter() + .map(|execution| vec![Vec::new(); physical.input_count(execution.candidate)]) + .collect(); + for (position, &(consumer, edge, _)) in selectable.edges.iter().enumerate() { + selectable.edges_into[consumer][physical.edge(edge).input_index_of_consumer] + .push(position); + } + selectable } - /// The id of `context`, added to the list when new. - fn context_id(&mut self, context: ExecutionContext) -> ExecutionContextId { - match self.contexts.iter().position(|known| *known == context) { + /// The id of `context` with `request_anchor` + /// ([`Self::request_anchor_of_context`]), added to the list when new. + fn context_id( + &mut self, + context: ExecutionContext, + request_anchor: Option<(AtModifier, EvaluationGrid)>, + ) -> ExecutionContextId { + match self + .contexts + .iter() + .zip(&self.request_anchor_of_context) + .position(|known| known == (&context, &request_anchor)) + { Some(index) => ExecutionContextId(index), None => { self.contexts.push(context); + self.request_anchor_of_context.push(request_anchor); ExecutionContextId(self.contexts.len() - 1) } } @@ -163,15 +223,13 @@ impl ExecutionContexts { if !producer.delivers_on(physical, &self.contexts[context.0]) { return None; } - let context = match ( - producer.physical_properties.lifetime(), - producer.physical_properties.timing(), - ) { - (Lifetime::AcrossEvaluations(_), _) => { - self.context_id(ExecutionContext::Materialized(u)) - } - (_, ExecutionTiming::IngestionTime) => self.context_id(EVERY_ARRIVING_ROW), - (Lifetime::None | Lifetime::OneEvaluation(_), ExecutionTiming::QueryTime) => context, + let properties = producer.physical_properties; + let context = if properties.is_across_evaluations() { + self.context_id(ExecutionContext::Materialized(u), None) + } else if properties.timing() == ExecutionTiming::IngestionTime { + self.context_id(EVERY_ARRIVING_ROW, None) + } else { + context }; Some(Execution { context, @@ -181,10 +239,11 @@ impl ExecutionContexts { /// The context the inputs of `consumer` run in: an `AcrossEvaluations` /// candidate's at ingestion time, the ingestion pipeline of its refresh - /// window (its chain width, else its group's evaluation grid); - /// a `TimeShift` anchored at `@ end()` or `@ start()` in an evaluation - /// context, the grid of its queries' request end or start times; else - /// the consumer's own context. + /// window (`PhysicalCandidate::own_schedule`); a `TimeShift` anchored at + /// `@ end()` or `@ start()` in an evaluation context, a context of its + /// own per anchor, on the grid of its queries' request end (start) + /// times: a request's start and end, and an evaluation, are different + /// times; else the consumer's own context. fn input_context( &mut self, physical: &PhysicalASAPMemoCandidates, @@ -194,27 +253,12 @@ impl ExecutionContexts { let candidate = &physical.physical_candidates[consumer.candidate.0]; let memo = &physical.logical_candidates.memo; let group = candidate.logical_candidate_id.group; - let at_ingestion = candidate.physical_properties.timing() == ExecutionTiming::IngestionTime; - if at_ingestion - && matches!( - candidate.physical_properties.lifetime(), - Lifetime::AcrossEvaluations(_) - ) + let properties = candidate.physical_properties; + if properties.timing() == ExecutionTiming::IngestionTime + && properties.is_across_evaluations() { - let refresh_window = match candidate.chain_tumbling_window_width_ms(physical) { - Some(width_ms) => Some(RefreshWindow { - width_ms, - phase_ms: 0, - }), - None => physical.logical_candidates.evaluation_grid_of_group[group.0 as usize].map( - |grid| RefreshWindow { - width_ms: grid.period_ms, - phase_ms: grid.phase_ms, - }, - ), - }; - return match refresh_window { - Some(window) => self.context_id(ExecutionContext::IngestionPipeline(window)), + return match candidate.own_schedule(physical) { + Some(window) => self.context_id(ExecutionContext::IngestionPipeline(window), None), None => consumer.context, }; } @@ -243,12 +287,28 @@ impl ExecutionContexts { }); match period_ms { 0 => consumer.context, - period_ms => self.context_id(ExecutionContext::Evaluation { - grid: EvaluationGrid { - period_ms, - phase_ms: 0, + period_ms => self.context_id( + ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms: 0, + }, }, - }), + Some((at, grid)), + ), + } + } + + /// Keeps, of the executions answering each query output, those `keep` + /// keeps (given the query, the execution's position and the + /// execution). + pub(crate) fn retain_answering( + &mut self, + mut keep: impl FnMut(QueryId, usize, Execution) -> bool, + ) { + let executions = &self.executions; + for (query, answering) in &mut self.answering { + answering.retain(|&index| keep(*query, index, executions[index])); } } @@ -257,23 +317,46 @@ impl ExecutionContexts { self.contexts[id.0] } + /// For the context with id `id` that runs the inputs of a `TimeShift` + /// anchored with `@ end()` or `@ start()`: the anchor, and the + /// evaluation grid of the queries whose requests it runs for. + pub(crate) fn request_anchor( + &self, + id: ExecutionContextId, + ) -> Option<(AtModifier, EvaluationGrid)> { + self.request_anchor_of_context[id.0] + } + /// Every execution reached from a query output, ascending. pub(crate) fn executions(&self) -> &[Execution] { &self.executions } /// Every $y_{\kappa,u,c,i}$ as (consumer execution, edge, producer - /// execution), ascending. - pub(crate) fn edges(&self) -> &[(Execution, PhysicalEdgeId, Execution)] { + /// execution), executions by position, ascending. + pub(crate) fn edges(&self) -> &[(usize, PhysicalEdgeId, usize)] { &self.edges } - /// The position of `execution` in [`Self::executions`]. - pub(crate) fn index_of(&self, execution: Execution) -> usize { - self.executions - .binary_search(&execution) - .expect("a reached execution") + /// The positions in [`Self::edges`] of the edges into input `input` of + /// the execution at position `consumer`. + pub(crate) fn edges_into(&self, consumer: usize, input: usize) -> &[usize] { + &self.edges_into[consumer][input] } + + /// The number of inputs of the execution at position `index`. + pub(crate) fn input_count(&self, index: usize) -> usize { + self.edges_into[index].len() + } +} + +/// Whether a candidate with `properties` has at most one consumer per +/// context: a query-time result that is not materialized ($\ell$ = None). +/// Consumers in one context that share a result read a `OneEvaluation` or a +/// materialized producer; the consumers of one ingestion run share its +/// rows. +pub(crate) fn has_one_reader_per_context(properties: PhysicalProperties) -> bool { + properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime } /// Each query output reference (a query, and the group its operator root, @@ -294,28 +377,11 @@ pub(crate) fn output_groups(physical: &PhysicalASAPMemoCandidates) -> Vec<(Query .collect() } -/// The evaluation grid of `entry`'s query, the grid of its context: the -/// times its evaluations fall on (`QueryRecurrence::evaluation_grid_ms` of -/// its range evaluation), anchored at the epoch. Times that are not known -/// may fall on any millisecond: a 1 ms grid, on which only producers stored -/// at every time deliver. +/// The evaluation grid of `entry`'s query, the grid of its context +/// ([`evaluation_grid_ms`]), anchored at the epoch. pub(crate) fn evaluation_grid(entry: &QueryWorkloadEntry) -> EvaluationGrid { EvaluationGrid { - period_ms: entry - .recurrence - .evaluation_grid_ms(entry.range_evaluation) - .unwrap_or(1), + period_ms: evaluation_grid_ms(entry), phase_ms: 0, } } - -/// The grid, in ms, of the times `entry`'s requests anchor `@ end()` or -/// `@ start()` (`at`) at -/// ([`asap_cost::plan::execution_demand::request_time_grid_ms`]). -pub(crate) fn request_time_grid_ms(entry: &QueryWorkloadEntry, at: AtModifier) -> u64 { - asap_cost::plan::execution_demand::request_time_grid_ms( - &entry.recurrence, - entry.range_evaluation, - at, - ) -} diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index 58e3bc768..e00962a0a 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -391,7 +391,7 @@ pub struct VectorMatch { /// PromQL `@` modifier — pins a selector's evaluation time to an anchor instead /// of the query evaluation time (issue #40). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub enum AtModifier { /// `@ start()` — the query range's start instant. Start, diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index 6b44df160..d0c17c376 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -3,6 +3,7 @@ use crate::ir::properties::ExecutionTiming; use crate::ir::{ASAPOp, Operator}; +use crate::plan::materialized_view::RefreshWindow; use crate::stages::{CandidateForMemoGroupId, ExecutionContext, PhysicalASAPMemoCandidates}; use crate::stages::{Lifetime, PhysicalProperties}; use crate::workload::DataArrival; @@ -50,10 +51,8 @@ impl PhysicalCandidate { /// /// `DataArrival::Unknown` counts as arriving. /// The times of a - /// `Materialized(v)` context are $v$'s own: every window end of a chain - /// $v$ (its width at phase 0), else its group's evaluation grid; with - /// neither they are not known, and no time-dependent producer delivers - /// on it. + /// `Materialized(v)` context are $v$'s own ([`Self::own_schedule`]); + /// when they are not known, no non-final producer delivers on it. pub fn delivers_on( &self, candidates: &PhysicalASAPMemoCandidates, @@ -81,15 +80,9 @@ impl PhysicalCandidate { (window.width_ms, window.phase_ms) } ExecutionContext::Materialized(v) => { - let materialized = &candidates.physical_candidates[v.0]; - match materialized.chain_tumbling_window_width_ms(candidates) { - Some(width_ms) => (width_ms, 0), - None => match logical.evaluation_grid_of_group - [materialized.logical_candidate_id.group.0 as usize] - { - Some(grid) => (grid.period_ms, grid.phase_ms), - None => return false, - }, + match candidates.physical_candidates[v.0].own_schedule(candidates) { + Some(window) => (window.width_ms, window.phase_ms), + None => return false, } } }; @@ -108,6 +101,25 @@ impl PhysicalCandidate { } } + /// The times it computes at when materialized across evaluations, on + /// its own schedule: every window end of a chain (its width at phase + /// 0), else its group's evaluation grid (`evaluation_grid_of_group`); + /// `None` when neither is known. + pub fn own_schedule(&self, candidates: &PhysicalASAPMemoCandidates) -> Option { + match self.chain_tumbling_window_width_ms(candidates) { + Some(width_ms) => Some(RefreshWindow { + width_ms, + phase_ms: 0, + }), + None => candidates.logical_candidates.evaluation_grid_of_group + [self.logical_candidate_id.group.0 as usize] + .map(|grid| RefreshWindow { + width_ms: grid.period_ms, + phase_ms: grid.phase_ms, + }), + } + } + /// Its tumbling-window width when it is a chain /// (`SummaryTumblingWindowChain`). pub fn chain_tumbling_window_width_ms( diff --git a/crates/types/src/stages/physical/properties/mod.rs b/crates/types/src/stages/physical/properties/mod.rs index e37dad27d..f59e9b4f9 100644 --- a/crates/types/src/stages/physical/properties/mod.rs +++ b/crates/types/src/stages/physical/properties/mod.rs @@ -86,6 +86,12 @@ impl PhysicalProperties { pub fn is_materialized(&self) -> bool { self.lifetime != Lifetime::None } + + /// Whether $\ell_v$ = `AcrossEvaluations`: its result is kept from one + /// evaluation to the next. Derived, not stored. + pub fn is_across_evaluations(&self) -> bool { + matches!(self.lifetime, Lifetime::AcrossEvaluations(_)) + } } #[cfg(test)] From b64b5fe1302e5f9e0e5accc705e08643e0467db3 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:29:05 +0000 Subject: [PATCH 04/23] docs(stage3): list exact_only.rs and milp/retention.rs, and (X), in the Stage 3 rows (#1207 PR 7) --- docs/design_docs/proposals/planner-layering.md | 2 +- docs/develop_docs/asap-aware-mapping-architecture.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 30e0db06d..ff616346c 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -511,7 +511,7 @@ candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `selectable.rs`, | Pass 1 | `pass1_memo` → `Memo`, the one entry: inside it, the computation targets (walking the memo groups, less the summaries the deployment does not support; `ComputationTarget` is Pass 1's private working type), then each target's ways to compute added as candidates of the groups, a summary with one build per configuration of its `parameter_list` | `stage1-pass1-per-computation-candidates/src/lib.rs`; `computation_targets.rs`; `candidates/` (`mod.rs`, `pending.rs`, `measure_output.rs`); `summary_realization.rs` builds the summary inputs (the sketch algorithms that serve an intent, each row's update) | | Memo types | `Memo`, `MemoGroup`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges; then each group's evaluation grid and the groups each query may read only exactly | `stage1-pass2-reuse-candidates/src/` (`lib.rs`, `rules/{candidates.rs,rollup/,topk.rs,chains.rs}`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, `evaluation_grids.rs`; the readout levels: `asap_accuracy::readout_levels`; the groups each query may read only exactly: `asap_accuracy::ErrorMetricsOfMemo::exact_only_groups_of_query`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | -| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `execution_contexts.rs` for $h_v$ and (R), `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | +| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `retention.rs` for $h_v$ and (R), `exact_only.rs` for each query's exact-only reads $a_{q,e}$ and (X), `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 373d4ad3a..d6fae1700 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -78,7 +78,7 @@ same pipeline behind the `OptimizationPass` trait. | What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `summary_type/capability.rs` | | Stage 2: the timings and lifetimes offered per logical candidate and their prune rules, the per-group offers (ingestion time, across evaluations), the physical edges, and the alternatives no plan can use | `asap-stage2-physical-optimization`: `candidate.rs`, `group.rs`, `edge.rs`, `pruning.rs`; the admissible-edge indexes in `asap-types`, `stages/physical/memo_candidates/mod.rs` | | Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state/mod.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | -| Stage 3: the execution contexts and the executions and edges a plan may select (`selectable.rs`), each execution's and each edge's cost and each retaining candidate's $\pi_v$ (`cost.rs`), how far back each edge reads its producer (`retention.rs`), the selection MILP over execution contexts with $h_v$ and (R) (`milp/`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | +| Stage 3: the execution contexts and the executions and edges a plan may select (`selectable.rs`), each execution's and each edge's cost and each retaining candidate's $\pi_v$ (`cost.rs`), how far back each edge reads its producer (`retention.rs`), the selection MILP over execution contexts with $h_v$ and (R), and with each query's exact-only reads $a_{q,e}$ and (X) (`milp/`, `exact_only.rs`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | | The accuracy rules: per-family guarantees, each operator's error contract, the metric rule, each readout's required accuracy, α^log of a logical edge | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `guarantee/`, `evidence.rs`, `error_contract/`, `metric_rule/`, `readout_caps/`, `logical_edge/`) | | The statistic a summary is read for an aggregation intent (`SketchStatistic::for_intent`) | `asap-types`, `summary_type/statistic.rs` | From ce0604148ada63f02c14dfbc16f17a6e26525e2e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:30:03 +0000 Subject: [PATCH 05/23] fix(stage3): the greedy bound counts the least retention an execution pays, pi_v times its least need over its uses (#1207 PR 7) --- .../src/milp/model/tests.rs | 38 ++++++++++++++ .../src/milp/upper_bound.rs | 51 +++++++++++++++++-- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index ce1ce3580..29f96ca3b 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -666,3 +666,41 @@ fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { assert_eq!(unused, vec![true, true]); assert!(solve_selection(&inputs).is_ok()); } + +/// `m[5m]` every 10 s over a scan that is not materialized (cost 1) or a +/// `OneEvaluation` one (cost 2, retaining at 1e6/s per second retained). +/// The greedy plan reads the first, at 2/s. The second's own cost alone is +/// no dearer than that, but (R) makes it retain the 5 minutes its one +/// consumer reads, 3e8/s: the bound leaves it out. +#[test] +fn the_bound_counts_the_least_retention_an_execution_pays() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = every_10s(1); + let (selectable, costs) = + selection_inputs(&physical, &entries, &[1.0, 2.0, 1.0], &[], &[(1, 1e6)]); + let left_out = dearer_than_a_feasible_plan(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }); + let left_out: Vec = selectable + .executions() + .iter() + .zip(&left_out) + .filter(|(_, &left_out)| left_out) + .map(|(execution, _)| execution.candidate.0) + .collect(); + assert_eq!(left_out, vec![1]); +} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index cf21eea51..46132285b 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -16,6 +16,7 @@ use std::collections::{BTreeMap, HashMap}; use super::exact_only::must_not_read; use super::model::{selection_cost_per_second, SelectionInputs}; +use crate::retention::output_retention_need_ms; use crate::selectable::{has_one_reader_per_context, QueryId}; /// How many times the greedy plan is rebuilt without an execution that @@ -28,9 +29,10 @@ type TreeInputs = Vec>; /// Per execution, by position in `SelectableExecutions::executions`, /// whether it costs more than a feasible plan -/// ([`feasible_plan_cost_per_second`]): its own cost plus the most its -/// input edges can take off ($\min(0, c_{\kappa,u,v,i})$ per input) is over -/// the bound. Such an execution is in no optimal plan, as long as every +/// ([`feasible_plan_cost_per_second`]): its own cost, plus the least +/// retention $\pi_v h_v$ (R) makes it pay, plus the most its input edges +/// can take off ($\min(0, c_{\kappa,u,v,i})$ per input) is over the +/// bound. Such an execution is in no optimal plan, as long as every /// other selected execution costs at least 0 with its selected inputs. /// None is ruled out when no feasible plan is found. pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec { @@ -39,19 +41,58 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec> = vec![None; count]; + let mut read = |index: usize, need_ms: u64| { + let least = &mut least_need_ms[index]; + *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); + }; + for (position, &(_, _, producer)) in selectable.edges().iter().enumerate() { + if let (Some(_), Some(need_ms)) = ( + costs.per_edge[position], + costs.retention_need_ms_per_edge[position], + ) { + read(producer, need_ms); + } + } + for (query, answering) in &selectable.answering { + for &index in answering { + read(index, output_retention_need_ms(&inputs.entries[query.0])); + } + } + let least_retention_cost = |index: usize| { + let v = executions[index].candidate; + match ( + costs.retention_cost_per_retained_second.get(&v), + least_need_ms[index], + ) { + (Some(per_retained_second), Some(need_ms)) + if !costs.raw_data_source.contains_key(&v) => + { + per_retained_second * need_ms as f64 / 1_000.0 + } + _ => 0.0, + } + }; let tolerance = 1e-9 * bound.abs().max(1e-12); (0..count) .map(|index| { - let Ok(own) = &inputs.costs.per_execution[index] else { + let Ok(own) = &costs.per_execution[index] else { return false; }; let least = own + + least_retention_cost(index) + (0..selectable.input_count(index)) .map(|i| { selectable .edges_into(index, i) .iter() - .filter_map(|&position| inputs.costs.per_edge[position]) + .filter_map(|&position| costs.per_edge[position]) .fold(0.0, f64::min) }) .sum::(); From a692ec2bb76ec492a884dfb097b5aba562f5651f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:37:12 +0000 Subject: [PATCH 06/23] perf(stage3): leave out a query-time execution with lifetime None that its OneEvaluation twin dominates (#1207 PR 7) --- .../src/milp/model/mod.rs | 11 +- .../src/milp/model/tests.rs | 47 ++++++++ .../src/milp/upper_bound.rs | 100 +++++++++++++++++- 3 files changed, 153 insertions(+), 5 deletions(-) diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index d21477e33..6f4c8aa35 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -30,7 +30,7 @@ use asap_types::workload::QueryWorkloadEntry; use super::exact_only::{exact_only_constraints, exact_only_variables}; use super::graph::Components; use super::retention::{least_retention_ms, retention_constraints, retention_variables}; -use super::upper_bound::dearer_than_a_feasible_plan; +use super::upper_bound::{dearer_than_a_feasible_plan, dominated_by_one_evaluation_twin}; use crate::cost::ExecutionCosts; use crate::selectable::{has_one_reader_per_context, Execution, QueryId, SelectableExecutions}; @@ -326,8 +326,10 @@ pub(crate) enum SelectionMilpError { /// y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s r_s$ ($\pi_v h_v$ of a /// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], /// [`raw_data_disk_constraints`], (R) and (X), with HiGHS at gap 0 on one -/// thread. The executions dearer than a feasible plan are left out first -/// ([`dearer_than_a_feasible_plan`]), with the $h_v$ of a candidate whose +/// thread. The executions dearer than a feasible plan +/// ([`dearer_than_a_feasible_plan`]) or dominated by their `OneEvaluation` +/// twin ([`dominated_by_one_evaluation_twin`]) are left out first, with +/// the $h_v$ of a candidate whose /// executions are all left out, then the coefficients kept are scaled so /// the largest is 1,000 (HiGHS's tolerances are absolute). pub(crate) fn solve_selection( @@ -337,8 +339,9 @@ pub(crate) fn solve_selection( let costs = inputs.costs; let kept: Vec = dearer_than_a_feasible_plan(inputs) .iter() + .zip(dominated_by_one_evaluation_twin(inputs)) .zip(&costs.per_execution) - .map(|(&left_out, cost)| cost.is_ok() && !left_out) + .map(|((&dearer, dominated), cost)| cost.is_ok() && !dearer && !dominated) .collect(); let components = Components::of(inputs); let mut problem = ProblemVariables::new(); diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 29f96ca3b..ab111bf71 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -704,3 +704,50 @@ fn the_bound_counts_the_least_retention_an_execution_pays() { .collect(); assert_eq!(left_out, vec![1]); } + +/// `m[5m]` every 10 s over a scan offered with ℓ = None (v0) and +/// `OneEvaluation` (v1). At equal cost and with the same consumer, v1 +/// dominates v0, which is left out; v0 is kept when v1 costs more, or when +/// v1 cannot be read by v0's consumer (its edge is not costed). +#[test] +fn a_none_execution_its_one_evaluation_twin_dominates_is_left_out() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = every_10s(1); + let dominated = |w: &[f64], twin_edge_costed: bool| { + let (selectable, mut costs) = selection_inputs(&physical, &entries, w, &[], &[]); + for (position, &(_, edge, _)) in selectable.edges().iter().enumerate() { + if physical.edge(edge).producer_candidate_id.0 == 1 && !twin_edge_costed { + costs.per_edge[position] = None; + } + } + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let candidates: Vec = selectable + .executions() + .iter() + .zip(dominated_by_one_evaluation_twin(&inputs)) + .filter(|(_, dominated)| *dominated) + .map(|(execution, _)| execution.candidate.0) + .collect(); + let solved = solve_selection(&inputs).unwrap(); + (candidates, self::candidates(&solved)) + }; + assert_eq!(dominated(&[1.0, 1.0, 1.0], true), (vec![0], vec![1, 2])); + assert_eq!(dominated(&[1.0, 2.0, 1.0], true), (vec![], vec![0, 2])); + assert_eq!(dominated(&[1.0, 1.0, 1.0], false), (vec![], vec![0, 2])); +} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index 46132285b..5fe4d6d27 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -11,8 +11,14 @@ //! plans became 4e-7, and the solver returned a plan 12.6% dearer. The //! greedy plan bounds the optimum by about 0.0088/s, so the chain is left //! out and the largest coefficient left is about 0.006/s. +//! +//! Also left out: an execution with $\ell$ = None that its `OneEvaluation` +//! twin dominates ([`dominated_by_one_evaluation_twin`]). + +use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::collections::{BTreeMap, HashMap}; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::stages::{CandidateForMemoGroupId, ExecutionContextId, Lifetime}; use super::exact_only::must_not_read; use super::model::{selection_cost_per_second, SelectionInputs}; @@ -23,6 +29,10 @@ use crate::selectable::{has_one_reader_per_context, QueryId}; /// made it infeasible before giving up on a bound. const REPAIRS: usize = 32; +/// Per (context, logical candidate), its query-time executions as +/// (execution, lifetime, cost per second). +type Twins = HashMap<(ExecutionContextId, CandidateForMemoGroupId), Vec<(usize, Lifetime, f64)>>; + /// Per input of an execution, the edge its cheapest tree reads it over, as /// (edge position, producer execution). type TreeInputs = Vec>; @@ -313,3 +323,91 @@ impl GreedyPlan { Ok(()) } } + +/// Per execution, by position in `SelectableExecutions::executions`, +/// whether it is a query-time execution with $\ell$ = None that its +/// `OneEvaluation` twin $b$ (the same logical candidate, so the same θ and +/// the same error bounds, in the same context) dominates: $b$ costs no +/// more, reads each of its producers on the same input at no more, is +/// read by each of its consumers on the same input at no more, and answers +/// every query output it answers. A plan selecting it selects $b$ in its +/// place at no more cost and stays feasible: $b$'s (F) is the looser +/// $x \le$ uses, its edges are the same ones (so (A), (B), (E) and (X) +/// hold as before), and (K), per execution, sees the same θ through the +/// same producers. A candidate with $h_v$ or raw data on disk is never +/// left out so, its retention depending on its own readers. +/// +/// Example: a `SummaryAgg` KLL build read by one readout in a context, +/// offered with $\ell$ = None and `OneEvaluation` at equal cost: the None +/// execution is left out. +pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> Vec { + let selectable = inputs.selectable; + let physical = inputs.physical; + let costs = inputs.costs; + let executions = selectable.executions(); + let count = executions.len(); + let no_more = |cost: f64, than: f64| cost <= than + 1e-12 * than.abs().max(1e-12); + // Per execution, its costed edges in, by (producer, input), and out, by + // (consumer, input). + let mut edges_in: Vec> = vec![HashMap::new(); count]; + let mut edges_out: Vec> = vec![HashMap::new(); count]; + for (position, &(consumer, edge, producer)) in selectable.edges().iter().enumerate() { + let Some(cost) = costs.per_edge[position] else { + continue; + }; + let input = physical.edge(edge).input_index_of_consumer; + edges_in[consumer].insert((producer, input), cost); + edges_out[producer].insert((consumer, input), cost); + } + let mut answers: Vec> = vec![BTreeSet::new(); count]; + for (query, answering) in &selectable.answering { + for &index in answering { + answers[index].insert(*query); + } + } + // The query-time executions of each logical candidate in each context, + // without $h_v$ or raw data, with their cost. + let mut twins: Twins = HashMap::new(); + for (index, execution) in executions.iter().enumerate() { + let v = execution.candidate; + let candidate = &physical.physical_candidates[v.0]; + let properties = candidate.physical_properties; + if properties.timing() != ExecutionTiming::QueryTime + || costs.retention_cost_per_retained_second.contains_key(&v) + || costs.raw_data_source.contains_key(&v) + { + continue; + } + let Ok(cost) = costs.per_execution[index] else { + continue; + }; + twins + .entry((execution.context, candidate.logical_candidate_id)) + .or_default() + .push((index, properties.lifetime(), cost)); + } + let mut dominated = vec![false; count]; + for members in twins.values() { + for &(a, lifetime, cost_a) in members { + if lifetime != Lifetime::None { + continue; + } + dominated[a] = members.iter().any(|&(b, lifetime, cost_b)| { + matches!(lifetime, Lifetime::OneEvaluation(_)) + && no_more(cost_b, cost_a) + && edges_in[a].iter().all(|(key, &cost)| { + edges_in[b] + .get(key) + .is_some_and(|&twin| no_more(twin, cost)) + }) + && edges_out[a].iter().all(|(key, &cost)| { + edges_out[b] + .get(key) + .is_some_and(|&twin| no_more(twin, cost)) + }) + && answers[a].is_subset(&answers[b]) + }); + } + } + dominated +} From 7972447ac22f096d775b3650b6116a83a57260e0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:45:25 +0000 Subject: [PATCH 07/23] perf(stage3): leave out (B) and (D) rows the one-reader equality (F) implies, and scale raw data on disk in its rows, per execution (#1207 PR 7) --- .../src/milp/model/mod.rs | 83 ++++++++++++------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 6f4c8aa35..369573969 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -156,7 +156,9 @@ fn input_satisfaction_constraints( constraints } -/// (B) $y_{\kappa,u,c,i} \le x$ of $u$'s execution it reads. +/// (B) $y_{\kappa,u,c,i} \le x$ of $u$'s execution it reads, except for +/// a producer with one reader per context ([`reads_once`]), whose (F) row +/// implies it. fn producer_selection_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, @@ -164,15 +166,16 @@ fn producer_selection_constraints( variables .y .iter() - .map(|(&position, &y)| { + .filter_map(|(&position, &y)| { let (_, _, producer) = inputs.selectable.edges()[position]; - constraint!(y <= variables.x[producer]) + (!reads_once(inputs, producer)).then(|| constraint!(y <= variables.x[producer])) }) .collect() } /// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x$ of $v$'s -/// execution answering $q$. +/// execution answering $q$, except for one with one reader per context +/// ([`reads_once`]), whose (F) row implies it. fn query_coverage_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, @@ -186,11 +189,23 @@ fn query_coverage_constraints( constraints.push(constraint!(covered == 1)); } for (&(_, execution), &z) in &variables.z { - constraints.push(constraint!(z <= variables.x[execution])); + if !reads_once(inputs, execution) { + constraints.push(constraint!(z <= variables.x[execution])); + } } constraints } +/// Whether the execution at position `index` has one reader per context +/// ([`has_one_reader_per_context`]): its (F) row is the equality +/// $\sum y + \sum_q z_{q,v} = x_{\kappa,v}$ ([`use_constraints`]), which +/// with $y, z \ge 0$ implies (B) and (D) for it, so those rows are left +/// out. +fn reads_once(inputs: &SelectionInputs<'_>, index: usize) -> bool { + let v = inputs.selectable.executions()[index].candidate; + has_one_reader_per_context(inputs.physical.physical_candidates[v.0].physical_properties) +} + /// (E) $t_u + 1 \le t_c + n(1 - y_{\kappa,u,c,i})$ between the producer's /// and the consumer's executions, only on edges inside a strongly connected /// component of the selectable edges, $n$ its size. @@ -229,14 +244,9 @@ fn use_constraints( uses_of(inputs, variables) .into_iter() .enumerate() - .map(|(index, used)| { - let v = inputs.selectable.executions()[index].candidate; - match has_one_reader_per_context( - inputs.physical.physical_candidates[v.0].physical_properties, - ) { - true => constraint!(used == variables.x[index]), - false => constraint!(variables.x[index] <= used), - } + .map(|(index, used)| match reads_once(inputs, index) { + true => constraint!(used == variables.x[index]), + false => constraint!(variables.x[index] <= used), }) .collect() } @@ -245,11 +255,13 @@ fn use_constraints( /// that keeps the most: $r_s \ge \pi_v h_v$ for each query-time `Scan` $v$ /// of source $s$ with $h_v$, and $r_s \ge d_v x_{\kappa,v}$ for each kept /// execution of one without, $d_v$ its cost per second of the raw data it -/// keeps retaining everything. +/// keeps retaining everything. The objective's scale is in the rows, so +/// $r_s$ is the scaled cost and its objective coefficient is 1. fn raw_data_disk_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, kept: &[bool], + objective_scale: f64, ) -> Vec { let costs = inputs.costs; let mut scans: Vec<(&PhysicalCandidateId, &Source)> = costs.raw_data_source.iter().collect(); @@ -259,11 +271,11 @@ fn raw_data_disk_constraints( let r = variables.raw_data_on_disk[source]; if let Some(&h) = variables.h.get(v) { let per_retained_second = costs.retention_cost_per_retained_second[v]; - constraints.push(constraint!(r >= per_retained_second * h)); + constraints.push(constraint!(r >= per_retained_second * objective_scale * h)); } else if let Some(&d) = costs.unbounded_raw_data_on_disk_cost_per_second.get(v) { for (index, execution) in inputs.selectable.executions().iter().enumerate() { if execution.candidate == *v && kept[index] { - constraints.push(constraint!(r >= d * variables.x[index])); + constraints.push(constraint!(r >= d * objective_scale * variables.x[index])); } } } @@ -346,19 +358,6 @@ pub(crate) fn solve_selection( let components = Components::of(inputs); let mut problem = ProblemVariables::new(); let variables = selection_variables(inputs, &kept, &components, &mut problem); - let constraints: Vec = [ - input_satisfaction_constraints(inputs, &variables), - producer_selection_constraints(inputs, &variables), - query_coverage_constraints(inputs, &variables), - acyclicity_constraints(inputs, &variables, &components), - use_constraints(inputs, &variables), - raw_data_disk_constraints(inputs, &variables, &kept), - retention_constraints(inputs, &variables.h, &variables.y, &variables.z), - exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), - ] - .into_iter() - .flatten() - .collect(); // The objective's terms per second, unscaled: (cost, variable). A // query-time scan's retention is in its source's r_s. let terms: Vec<(f64, Variable)> = costs @@ -381,7 +380,6 @@ pub(crate) fn solve_selection( .filter(|(v, _)| !costs.raw_data_source.contains_key(v)) .map(|(v, &h)| (costs.retention_cost_per_retained_second[v], h)), ) - .chain(variables.raw_data_on_disk.values().map(|&r| (1.0, r))) .collect(); // Scaled by the largest coefficient kept, the raw-data bounds of the // scans kept included. @@ -413,7 +411,30 @@ pub(crate) fn solve_selection( true => 1_000.0 / largest, false => 1.0, }; - let objective: Expression = terms.iter().map(|&(cost, v)| cost * scale * v).sum(); + let constraints: Vec = [ + input_satisfaction_constraints(inputs, &variables), + producer_selection_constraints(inputs, &variables), + query_coverage_constraints(inputs, &variables), + acyclicity_constraints(inputs, &variables, &components), + use_constraints(inputs, &variables), + raw_data_disk_constraints(inputs, &variables, &kept, scale), + retention_constraints(inputs, &variables.h, &variables.y, &variables.z), + exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), + ] + .into_iter() + .flatten() + .collect(); + // The raw data on disk is scaled by its rows ([`raw_data_disk_constraints`]). + let objective: Expression = terms + .iter() + .map(|&(cost, v)| cost * scale * v) + .chain( + variables + .raw_data_on_disk + .values() + .map(|&r| Expression::from(r)), + ) + .sum(); let variable_count = problem.len(); // HiGHS's columns are the variables in the order they were added. let column_of: HashMap = problem From 8c8a6741fa36ab72ea4c20ebc26350ff2a6675d5 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:08:10 +0000 Subject: [PATCH 08/23] fix(stage3): an anchored input runs once per request of the queries that reach its shift (#1207 PR 7) --- .../tests/stage3_objective.rs | 26 ++++++++++++++ crates/stage3-plan-selection/src/cost.rs | 26 +++++++------- .../stage3-plan-selection/src/selectable.rs | 36 +++++++++++-------- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index 3a95d9998..b8e6ea1b7 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -441,3 +441,29 @@ fn an_at_start_subtree_shares_no_evaluation_result_with_an_instant_query() { "{instant:?} and {at_start:?} share a node: {plan:#?}" ); } + +/// `@ end()` and `@ start()` range queries every 300 s at 30 s steps next +/// to the plain `sum_over_time(m[5m])` every 30 s, on the same 30 s grid: +/// an anchored input runs once per request of the queries that reach its +/// shift, not of every query on the grid reaching `m[5m]` (the objective +/// was 0.2404/s for a plan `plan_cost` charged 0.1364/s). +#[test] +fn the_objective_is_the_plan_cost_on_anchored_and_plain_queries_on_one_grid() { + let w = over_m(vec![ + requested( + "sum_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + requested( + "max_over_time(m[5m] @ start())", + 300_000, + Some((30_000, 600_000)), + ), + requested("sum_over_time(m[5m])", 30_000, None), + ]); + assert_equal( + "anchored and plain queries on one grid", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 753c2172a..de3179d63 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -473,7 +473,8 @@ fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidat /// the queries on its grid that reach the execution's group /// ([`context_executions_per_second`], which `plan_cost` uses too): at /// their evaluations, or, for the inputs of a `TimeShift` anchored with -/// `@`, once per request of the queries on the anchored evaluation grid; +/// `@`, once per request of the queries on the shift's grid that reach the +/// shift; /// in `Materialized(m)`, as often as $m$ runs /// ([`asap_cost::plan::execution_demand::executions_per_second`]); in the /// ingestion pipeline, once per second of arriving rows. @@ -496,18 +497,19 @@ pub(crate) fn executions_per_second( .filter_map(|&query| entries.get(query)) }; let rate = match selectable.context(execution.context) { - ExecutionContext::Evaluation { grid } => { - let (anchor, queries_grid) = match selectable.request_anchor(execution.context) { - Some((at, queries_grid)) => (Some(at), queries_grid), - None => (None, grid), - }; - context_executions_per_second( - anchor, - reaching(execution.candidate) - .filter(|entry| evaluation_grid(entry) == queries_grid), + ExecutionContext::Evaluation { grid } => match selectable.request_anchor(execution.context) + { + Some((at, queries)) => context_executions_per_second( + Some(at), + queries.iter().filter_map(|query| entries.get(query.0)), non_repeating_query_horizon_seconds, - ) - } + ), + None => context_executions_per_second( + None, + reaching(execution.candidate).filter(|entry| evaluation_grid(entry) == grid), + non_repeating_query_horizon_seconds, + ), + }, ExecutionContext::Materialized(m) => { let properties = physical.physical_candidates[m.0].physical_properties; rate_of_materialized( diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index 7dc151446..74d07e30c 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -72,9 +72,10 @@ pub(crate) struct SelectableExecutions { contexts: Vec, /// Per context, by [`ExecutionContextId`]: for the inputs of a /// `TimeShift` anchored with `@ end()` or `@ start()`, the anchor and - /// the evaluation grid of the queries whose requests they run for; - /// `None` for any other context. - request_anchor_of_context: Vec>, + /// the queries whose requests they run for (those on the shift's + /// evaluation grid that reach it), ascending; `None` for any other + /// context. + request_anchor_of_context: Vec)>>, /// Every execution reached from a query output, ascending. executions: Vec, /// Every $y_{\kappa,u,c,i}$ as (consumer execution, edge $(u,c,i)$, @@ -191,7 +192,7 @@ impl SelectableExecutions { fn context_id( &mut self, context: ExecutionContext, - request_anchor: Option<(AtModifier, EvaluationGrid)>, + request_anchor: Option<(AtModifier, Vec)>, ) -> ExecutionContextId { match self .contexts @@ -277,14 +278,19 @@ impl SelectableExecutions { else { return consumer.context; }; - let period_ms = memo.groups()[group.0 as usize] + let queries: Vec = memo.groups()[group.0 as usize] .reached_by_queries() .iter() - .filter_map(|&query| entries.get(query)) - .filter(|entry| evaluation_grid(entry) == grid) - .fold(0, |period_ms, entry| { - greatest_common_divisor(period_ms, request_time_grid_ms(entry, at)) - }); + .filter(|&&query| { + entries + .get(query) + .is_some_and(|entry| evaluation_grid(entry) == grid) + }) + .map(|&query| QueryId(query)) + .collect(); + let period_ms = queries.iter().fold(0, |period_ms, query| { + greatest_common_divisor(period_ms, request_time_grid_ms(&entries[query.0], at)) + }); match period_ms { 0 => consumer.context, period_ms => self.context_id( @@ -294,7 +300,7 @@ impl SelectableExecutions { phase_ms: 0, }, }, - Some((at, grid)), + Some((at, queries)), ), } } @@ -318,13 +324,15 @@ impl SelectableExecutions { } /// For the context with id `id` that runs the inputs of a `TimeShift` - /// anchored with `@ end()` or `@ start()`: the anchor, and the - /// evaluation grid of the queries whose requests it runs for. + /// anchored with `@ end()` or `@ start()`: the anchor, and the queries + /// whose requests it runs for. pub(crate) fn request_anchor( &self, id: ExecutionContextId, - ) -> Option<(AtModifier, EvaluationGrid)> { + ) -> Option<(AtModifier, &[QueryId])> { self.request_anchor_of_context[id.0] + .as_ref() + .map(|(at, queries)| (*at, queries.as_slice())) } /// Every execution reached from a query output, ascending. From a154b310c3c36ae81dec653403b870cfe5a6ee3f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:32:01 +0000 Subject: [PATCH 09/23] fix(stage3): an ingestion-time view's pi_v is one more output's cost over the window it maintains (#1207 PR 7) --- .../tests/stage3_objective.rs | 38 +++++++++++ crates/stage3-plan-selection/src/cost.rs | 66 +++++++++++-------- 2 files changed, 78 insertions(+), 26 deletions(-) diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index b8e6ea1b7..256ad0407 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -467,3 +467,41 @@ fn the_objective_is_the_plan_cost_on_anchored_and_plain_queries_on_one_grid() { objective_and_plan_cost(&w, &CostInputs::default(), true), ); } + +/// Example 3a of the planner-layering design doc: five one-off p99s over +/// 1,000,000 series sampled every 15 s, over the last 5 y, 1 y, 1 y offset +/// 1 y and 2 y, and 3 y offset 2 y, accuracy guarantees ignored. An +/// ingestion-time view behind an offset retains one output per window it +/// maintains, so its $\pi_v$ is one output's cost over that window: costed +/// per second instead, $\pi_v h_v$ of a view kept for years was 6.6e12/s, +/// its $W^{\mathrm{mat}}_v$ as negative, and the objective's scale left the +/// solver at a plan costing 4.68e7/s instead of 2.17e4/s. +#[test] +fn the_objective_is_the_plan_cost_on_one_off_queries_over_years() { + let query = |query: &str| { + serde_json::json!({ + "query": query, + "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.005, "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, + "predictability": "ad_hoc", "invocations": 1, "execute_at": 1767225600000_u64, + }) + }; + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": {"language": "promql", "query_batch": [ + query("quantile_over_time(0.99, latency_ms[5y])"), + query("quantile_over_time(0.99, latency_ms[1y])"), + query("quantile_over_time(0.99, latency_ms[1y] offset 1y)"), + query("quantile_over_time(0.99, latency_ms[1y] offset 2y)"), + query("quantile_over_time(0.99, latency_ms[3y] offset 2y)"), + ]}, + "data_workload": { + "arrival": {"mixed": {"rows_per_second": 1_000_000.0 / 15.0}}, + "sources": {"latency_ms": {"series_count": 1_000_000, "sample_interval": 15_000}}, + }, + })) + .unwrap(); + let models = PlanningModels::builtin().with_ignore_accuracy_guarantees(true); + let (objective, plan_cost) = + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1; + assert_equal("one-off queries over years", (objective, plan_cost)); + assert!(objective < 30_000.0, "objective {objective}"); +} diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index de3179d63..9012b723f 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -41,7 +41,7 @@ //! (`PhysicalASAPDAGNode::retains_rows`) is costed retaining the most its //! consumers read back ([`edge_retention_need_ms`]), the largest $h_v$ (R) //! allows. Retention is linear in that amount: $\pi_v$ is what retaining -//! one more window (or second) costs there, and $W^{\mathrm{mat}}_v$ of a +//! one more output (a window, or a second) costs there, and $W^{\mathrm{mat}}_v$ of a //! materialized candidate is its cost less $\pi_v h_v$. A query-time //! scan's retention is raw data on disk, kept once per source for the //! selected scan that keeps the most: $r_s \ge \pi_v h_v$ @@ -321,8 +321,10 @@ pub(crate) fn execution_costs( .map(|(index, execution)| (execution.candidate, index)) .collect(); // π_v of a candidate retaining rows every consumer reads within a bound: - // what one more Δ ms of retention costs, per second retained (Δ a - // window of a chain, else a second), at its retention's largest; its + // what one more Δ ms of retention costs, per second retained (Δ the + // time between two of its outputs: a chain's window, the window an + // ingestion-time view maintains, else a second; a retention is a whole + // number of outputs), at its retention's largest; its // whole own cost when it is materialized, its raw data on disk for a // query-time scan (each of its runs reads what it reads at that // retention, ρ_κ c_{κ,v}). @@ -331,10 +333,13 @@ pub(crate) fn execution_costs( let Some(retention_ms) = most_read_ms else { continue; }; - let delta_ms = physical.physical_candidates[v.0] - .chain_tumbling_window_width_ms(physical) - .unwrap_or(1_000) - .max(1); + let candidate = &physical.physical_candidates[v.0]; + let delta_ms = match candidate.physical_properties.timing() { + ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), + ExecutionTiming::QueryTime => candidate.chain_tumbling_window_width_ms(physical), + } + .unwrap_or(1_000) + .max(1); let longer = node_retaining(v, Some(retention_ms + delta_ms)); let delta_seconds = delta_ms as f64 / 1_000.0; let per_retained_second = match is_across_evaluations(physical, v) { @@ -622,32 +627,41 @@ fn cost_node<'a>( data: &DataWorkload, cost_inputs: &'a CostInputs, ) -> Result, String> { - let memo = &physical.logical_candidates.memo; - let group = physical.physical_candidates[v.0].logical_candidate_id.group; let window_ms = match node.output_state.timing { ExecutionTiming::QueryTime => scan_read_ms, - // A chain maintains its own window; any other grouping the first - // range below its group. - ExecutionTiming::IngestionTime => physical.physical_candidates[v.0] - .chain_tumbling_window_width_ms(physical) - .or_else(|| { - maintained_window_ms( - group, - |group: MemoGroupId| &memo.groups()[group.0 as usize].expression, - |group| { - memo.groups()[group.0 as usize] - .expression - .children() - .first() - .map(|g| **g) - }, - ) - }), + ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), }; node_resource_usage_and_cost(node, inputs, (1, per_second), window_ms, data, cost_inputs) .map_err(|error| error.to_string()) } +/// The window ingestion-time candidate `v` maintains, the time between +/// two of its outputs: a chain its own window, any other grouping the +/// first range or chain below its group ([`asap_cost::maintained_window_ms`], +/// over the memo's groups); `None` without one (one output per second). +fn ingestion_window_ms( + physical: &PhysicalASAPMemoCandidates, + v: PhysicalCandidateId, +) -> Option { + let memo = &physical.logical_candidates.memo; + let group = physical.physical_candidates[v.0].logical_candidate_id.group; + physical.physical_candidates[v.0] + .chain_tumbling_window_width_ms(physical) + .or_else(|| { + maintained_window_ms( + group, + |group: MemoGroupId| &memo.groups()[group.0 as usize].expression, + |group| { + memo.groups()[group.0 as usize] + .expression + .children() + .first() + .map(|g| **g) + }, + ) + }) +} + /// A candidate's cost per second from its usage under cost model 1: its /// own, and the raw data a query-time scan keeps on disk /// ([`asap_cost::raw_data_disk_cost_per_second`]). From 51e8b283a76d0091a98ad333385de12af4c57d2c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:48:39 +0000 Subject: [PATCH 10/23] fix(stage3,cost): an instant query's @ end() input runs in its evaluation context (#1207 PR 7) An instant query's request ends at its evaluation, so the input of an `@ end()` shift read by an instant query is computed in the evaluation context, shared with the plain reads on its grid, in Stage 3's contexts and in `plan_cost` alike (`anchor_is_evaluation_time`). Range queries and `@ start()` keep their request contexts. --- crates/cost/src/plan/execution_demand/mod.rs | 72 +++++++++++++++++-- .../cost/src/plan/execution_demand/tests.rs | 18 +++++ .../tests/stage3_objective.rs | 19 +++++ .../stage3-plan-selection/src/selectable.rs | 14 +++- 4 files changed, 115 insertions(+), 8 deletions(-) diff --git a/crates/cost/src/plan/execution_demand/mod.rs b/crates/cost/src/plan/execution_demand/mod.rs index c07259f98..b692afe08 100644 --- a/crates/cost/src/plan/execution_demand/mod.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -306,7 +306,10 @@ pub enum Consumer { /// then the materialized nodes in `dag`'s node order. A `OneEvaluation` /// node is computed within its consumers' context, so a walk goes through /// it. -pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap> { +pub fn consumers( + dag: &PhysicalASAPDAG, + entries: &[QueryWorkloadEntry], +) -> HashMap> { let materialized_nodes: Vec = dag .nodes .iter() @@ -332,7 +335,40 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap None, }) .collect(); - let walks = query_starts(dag) + let starts = query_starts(dag); + // The queries reaching each anchored shift, through any node. + let mut reaching_shift: HashMap> = HashMap::new(); + for (query, start) in starts.iter().enumerate() { + let mut seen = HashSet::new(); + let mut stack = start.clone(); + while let Some(id) = stack.pop() { + if seen.insert(id) { + if request_anchor.contains_key(&id) { + reaching_shift.entry(id).or_default().insert(query); + } + stack.extend(inputs(id)); + } + } + } + // Whether query `query`'s input of shift `shift` anchored with `at` + // runs in its evaluation context ([`anchor_is_evaluation_time`] over + // the queries on its grid reaching the shift). + let in_evaluation_context = |shift: PhysicalASAPNodeId, query: usize, at: AtModifier| { + let Some(entry) = entries.get(query) else { + return false; + }; + let grid_ms = evaluation_grid_ms(entry); + anchor_is_evaluation_time( + at, + reaching_shift + .get(&shift) + .into_iter() + .flatten() + .filter_map(|&q| entries.get(q)) + .filter(|other| evaluation_grid_ms(other) == grid_ms), + ) + }; + let walks = starts .into_iter() .enumerate() .map(|(query, start)| (Consumer::Query(query), start)) @@ -352,9 +388,12 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap Consumer::QueryRequest { query, at }, + (Consumer::Query(query), Some(&at)) if !in_evaluation_context(id, query, at) => { + Consumer::QueryRequest { query, at } + } (consumer, _) => consumer, }; stack.extend(inputs(id).map(|input| (input, below))); @@ -363,6 +402,29 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap( + at: AtModifier, + queries: impl IntoIterator, +) -> bool { + at == AtModifier::End + && queries.into_iter().any(|entry| { + entry + .range_evaluation + .is_none_or(|range| range.evaluations_per_request() <= 1) + }) +} + /// The grid, in ms, every evaluation of `entry`'s query falls on /// (`QueryRecurrence::evaluation_grid_ms` of its range evaluation), the /// grid of its evaluation context; 1 ms when not known, a grid on which @@ -516,7 +578,7 @@ pub fn computations( rate_of_materialized.insert(id, rate); } let mut count: HashMap<_, (u64, f64)> = HashMap::new(); - for (id, of_node) in consumers(dag) { + for (id, of_node) in consumers(dag, entries) { let entry = count.entry(id).or_default(); // The queries reaching it, per context: (evaluation grid, anchor). let mut queries_by_context: BTreeMap<(u64, Option), BTreeSet> = diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs index 27e13d846..5f5e4093b 100644 --- a/crates/cost/src/plan/execution_demand/tests.rs +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -403,3 +403,21 @@ fn invalid_recurrence_is_rejected() { Err(AnalyticalCostError::InvalidRecurrence) )); } + +/// `m[5m]` below an `@ end()` shift read by an instant query every 60 s, +/// and read directly by another instant query every 60 s: the anchor is +/// the evaluation time, so the range and the scan are computed in the one +/// evaluation context, once per minute. +#[test] +fn an_instant_at_end_input_runs_in_the_evaluation_context() { + let rates = computations( + &anchored_dag(AtModifier::End, vec![2, 1]), + &entries(&[fixed(60_000, 0), fixed(60_000, 0)], None), + 3600.0, + ) + .unwrap(); + for id in [0, 1] { + assert_eq!(rates[&id].0, 1); + close(rates[&id].1, 1.0 / 60.0); + } +} diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index 256ad0407..a84d97285 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -505,3 +505,22 @@ fn the_objective_is_the_plan_cost_on_one_off_queries_over_years() { assert_equal("one-off queries over years", (objective, plan_cost)); assert!(objective < 30_000.0, "objective {objective}"); } + +/// `sum_over_time(m[5m] @ end())` and `sum_over_time(m[5m])`, both instant +/// every 60 s: an instant query's `@ end()` is its evaluation time, so the +/// anchored input runs in the evaluation context and shares `m[5m]` with +/// the plain query, in the MILP and in `plan_cost` alike (0.0360/s; in a +/// request context of its own it cost 0.0620/s). +#[test] +fn an_instant_at_end_input_shares_the_evaluation_context() { + let w = over_m(vec![ + requested("sum_over_time(m[5m] @ end())", 60_000, None), + requested("sum_over_time(m[5m])", 60_000, None), + ]); + let (objective, plan_cost) = objective_and_plan_cost(&w, &CostInputs::default(), true); + assert_equal( + "instant @ end() beside a plain query", + (objective, plan_cost), + ); + assert!(objective < 0.04, "objective {objective}"); +} diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index 74d07e30c..22caea1cd 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -22,14 +22,18 @@ //! on the grid of the request end (start) times, once per request of a //! range query, in a context of its own per anchor: a request's start, //! its end and an evaluation are different times, so their results are -//! not shared. +//! not shared. An `@ end()` read by an instant query is anchored at its +//! evaluation time, so its input runs in the evaluation context +//! (`asap_cost::plan::execution_demand::anchor_is_evaluation_time`). //! - An input of a consumer in $\kappa$ reads a producer, over an admissible //! edge, only where the producer delivers on $\kappa$ //! (`PhysicalCandidate::delivers_on`). use std::collections::BTreeSet; -use asap_cost::plan::execution_demand::{evaluation_grid_ms, request_time_grid_ms}; +use asap_cost::plan::execution_demand::{ + anchor_is_evaluation_time, evaluation_grid_ms, request_time_grid_ms, +}; use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; @@ -244,7 +248,8 @@ impl SelectableExecutions { /// `@ end()` or `@ start()` in an evaluation context, a context of its /// own per anchor, on the grid of its queries' request end (start) /// times: a request's start and end, and an evaluation, are different - /// times; else the consumer's own context. + /// times (unless the anchor is the queries' evaluation time, + /// `anchor_is_evaluation_time`); else the consumer's own context. fn input_context( &mut self, physical: &PhysicalASAPMemoCandidates, @@ -288,6 +293,9 @@ impl SelectableExecutions { }) .map(|&query| QueryId(query)) .collect(); + if anchor_is_evaluation_time(at, queries.iter().map(|query| &entries[query.0])) { + return consumer.context; + } let period_ms = queries.iter().fold(0, |period_ms, query| { greatest_common_divisor(period_ms, request_time_grid_ms(&entries[query.0], at)) }); From 60b0388757bd626812450c37c623fd20d6791966 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:10:16 +0000 Subject: [PATCH 11/23] fix(cost): an @ start() input runs once per request of each distinct schedule and span (#1207 PR 7) --- crates/cost/src/plan/execution_demand/mod.rs | 35 +++++++++++++++---- .../cost/src/plan/execution_demand/tests.rs | 16 +++++++-- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/crates/cost/src/plan/execution_demand/mod.rs b/crates/cost/src/plan/execution_demand/mod.rs index b692afe08..9d038555a 100644 --- a/crates/cost/src/plan/execution_demand/mod.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -455,19 +455,42 @@ pub fn request_time_grid_ms(entry: &QueryWorkloadEntry, at: AtModifier) -> u64 { /// per execution of the context; Stage 3 and [`computations`] both cost it /// so): with no `anchor`, at every evaluation of theirs, a range query's /// steps each one ([`evaluation_rate`]); below a `TimeShift` anchored with -/// `anchor` (`@ end()`, `@ start()`), once per distinct anchor time of -/// their requests: a request's end, or its end less its range-evaluation -/// span. +/// `anchor`, once per request: below `@ end()` once per distinct request +/// end; below `@ start()` once per distinct (request schedule, span), as +/// two queries requested together with different spans need the input at +/// two start times, and a result not kept across evaluations serves only +/// the request it is computed for. /// /// Example: two range queries every 60 s at phase 0 with steps of 30 s, -/// spans 1 h and 90 s, below `@ start()`: their requests start at -/// `k · 60 s` and `k · 60 s + 30 s`, 2/60 per second; below `@ end()` -/// both end at `k · 60 s`, 1/60 per second. +/// spans 1 h and 2 m: below `@ start()` 2/60 per second (one run per +/// request of each); below `@ end()` both end at `k · 60 s`, 1/60 per +/// second. pub fn context_executions_per_second<'e>( anchor: Option, queries: impl IntoIterator, non_repeating_query_horizon_seconds: f64, ) -> Result { + let queries: Vec<&QueryWorkloadEntry> = queries.into_iter().collect(); + if anchor == Some(AtModifier::Start) { + // One run per request of each distinct (schedule, span). + let mut runs: Vec<(&QueryRecurrence, Option)> = Vec::new(); + let mut rate = 0.0; + for entry in queries { + let key = ( + &entry.recurrence, + entry.range_evaluation.map(|range| range.span_ms), + ); + if runs.contains(&key) { + continue; + } + runs.push(key); + rate += evaluation_rate( + [(&anchored_request_recurrence(entry, AtModifier::Start), None)], + non_repeating_query_horizon_seconds, + )?; + } + return Ok(rate); + } let per_evaluation: Vec<(QueryRecurrence, Option)> = queries .into_iter() .map(|entry| match anchor { diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs index 5f5e4093b..af83a6a66 100644 --- a/crates/cost/src/plan/execution_demand/tests.rs +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -356,9 +356,11 @@ fn a_shared_one_evaluation_at_end_shift_runs_its_input_once_per_request() { /// Two range queries every 60 s at a 30 s step, over 1 h and over 90 s, /// below one `@ start()` shift: one context (one evaluation grid, one /// anchor), whose requests start at `k · 60 s` and `k · 60 s + 30 s`: the -/// input runs 2/60 per second. +/// input runs 2/60 per second. Over 1 h and over 2 m, both start at +/// `k · 60 s` but at different times of one request: still 2/60. The same +/// query twice runs once. #[test] -fn at_start_inputs_run_once_per_distinct_request_start() { +fn at_start_inputs_run_once_per_request_of_each_span() { let with_span = |span_ms| QueryWorkloadEntry { query: asap_types::workload::Query("q".into()), requirements: Default::default(), @@ -384,6 +386,16 @@ fn at_start_inputs_run_once_per_distinct_request_start() { context_executions_per_second(Some(AtModifier::End), &entries, 3600.0).unwrap(), 1.0 / 60.0, ); + let same_start = [with_span(3_600_000), with_span(120_000)]; + close( + context_executions_per_second(Some(AtModifier::Start), &same_start, 3600.0).unwrap(), + 2.0 / 60.0, + ); + let twice = [with_span(120_000), with_span(120_000)]; + close( + context_executions_per_second(Some(AtModifier::Start), &twice, 3600.0).unwrap(), + 1.0 / 60.0, + ); } // Invalid fixed intervals and estimated rates still produce the evaluator's error. From 902d1c1a4806a78a69aa514dc3ba8b8d630a93b8 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:17:11 +0000 Subject: [PATCH 12/23] refactor(stage3,cost): one walk of retained reads, one rule for a context's queries, dominance in its own module (#1207 PR 7) - `crate::retention::reads_back`: every read of an execution's rows (edge needs and query-output spans), folded by costing, (R), `least_retention_ms` and the greedy bound; `ms_to_seconds` and `ExecutionCosts::retention_cost_per_second` for the ms-to-seconds step. - `asap_cost::plan::execution_demand::queries_of_context`: Stage 3 and `computations` pick a context's queries with it. - `milp/dominance.rs` holds `dominated_by_one_evaluation_twin`, over two lists per (context, logical candidate) and the selectable edges. - `SelectableExecutions`: `SelectableEdge`, `edges_out_of` next to `edges_into` (used by the components, (X) and dominance), private fields with getters, a map of context ids. - `raw_data_disk_rows` gives the raw-data rows to the scale and the constraints; `has_one_reader`; the scaled `raw_data_on_disk` doc. - `delivers_on` compares with `own_schedule` (no `% 0`). - asap-cost: the `Scan` arm and the retained raw rows share `raw_rows`. --- crates/cost/src/node/usage/mod.rs | 21 ++- crates/cost/src/plan/execution_demand/mod.rs | 66 +++++--- .../tests/stage3_objective.rs | 15 ++ crates/stage3-plan-selection/src/cost.rs | 81 ++++++---- .../stage3-plan-selection/src/cost/tests.rs | 16 +- crates/stage3-plan-selection/src/lib.rs | 2 +- .../src/milp/dominance.rs | 105 +++++++++++++ .../src/milp/exact_only.rs | 35 +++-- .../stage3-plan-selection/src/milp/graph.rs | 49 +++--- crates/stage3-plan-selection/src/milp/mod.rs | 4 +- .../src/milp/model/mod.rs | 113 +++++++------- .../src/milp/model/tests.rs | 10 +- .../src/milp/retention.rs | 67 ++++---- .../src/milp/upper_bound.rs | 146 +++--------------- crates/stage3-plan-selection/src/retention.rs | 53 ++++++- .../stage3-plan-selection/src/selectable.rs | 107 ++++++++----- crates/types/src/stages/physical/candidate.rs | 21 +-- 17 files changed, 519 insertions(+), 392 deletions(-) create mode 100644 crates/stage3-plan-selection/src/milp/dominance.rs diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index af859b6c5..c758624b2 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -359,9 +359,9 @@ pub fn node_resource_usage_and_cost<'a>( } _ => false, }; - // The raw rows of its sources back `lookback_ms` (every row held - // without one): by arrival rate, at most the rows they hold. - let retained_raw_rows = |lookback_ms: Option| -> Option { + // The raw rows of its sources back `lookback_ms`: (by arrival rate, + // when known; the rows they hold, when declared). + let raw_rows = |lookback_ms: Option| -> (Option, Option) { let by_rate = rows_per_ms .zip(lookback_ms) .map(|(rows_per_ms, lookback_ms)| (rows_per_ms * lookback_ms as f64).round() as u64); @@ -369,7 +369,12 @@ pub fn node_resource_usage_and_cost<'a>( .iter() .map(|source| data.sources.get(*source).and_then(|facts| facts.rows_held)) .sum(); - match (by_rate, rows_held) { + (by_rate, rows_held) + }; + // The raw rows its sources keep back `lookback_ms` (every row held + // without one): by arrival rate, at most the rows they hold. + let retained_raw_rows = |lookback_ms: Option| -> Option { + match raw_rows(lookback_ms) { (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), (by_rate, rows_held) => by_rate.or(rows_held), } @@ -496,7 +501,6 @@ pub fn node_resource_usage_and_cost<'a>( // it, at most the rows the source holds (all of them // without a window). let source = source_name(source); - let rows_held = data.sources.get(source).and_then(|facts| facts.rows_held); // At ingestion, one second of rows; at query time, one // execution reads the most its consumers read (`window_ms`), // else what the scan retains (its `retention_ms`, $h_v$); @@ -505,11 +509,6 @@ pub fn node_resource_usage_and_cost<'a>( true => Some(ONE_SECOND_MS), false => window_ms.or(node.retention_ms), }; - let by_rate = |lookback: Option| { - rows_per_ms.zip(lookback).map(|(rows_per_ms, lookback)| { - (rows_per_ms * lookback as f64).round() as u64 - }) - }; // The raw rows a query-time scan, or an ingestion-time scan // materialized across evaluations, keeps: back to the // oldest row read, its `retention_ms` ($h_v$; an offset @@ -517,7 +516,7 @@ pub fn node_resource_usage_and_cost<'a>( if !ingestion || raw_rows_materialized_at_ingestion { raw_retained_rows = retained_raw_rows(node.retention_ms); } - let by_rate = by_rate(lookback); + let (by_rate, rows_held) = raw_rows(lookback); let rows = match (ingestion, by_rate, rows_held) { (true, by_rate, _) => by_rate.ok_or_else(|| missing_rate(source))?, (false, Some(by_rate), Some(rows_held)) => by_rate.min(rows_held), diff --git a/crates/cost/src/plan/execution_demand/mod.rs b/crates/cost/src/plan/execution_demand/mod.rs index 9d038555a..9c336edb6 100644 --- a/crates/cost/src/plan/execution_demand/mod.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -357,16 +357,12 @@ pub fn consumers( let Some(entry) = entries.get(query) else { return false; }; - let grid_ms = evaluation_grid_ms(entry); - anchor_is_evaluation_time( - at, - reaching_shift - .get(&shift) - .into_iter() - .flatten() - .filter_map(|&q| entries.get(q)) - .filter(|other| evaluation_grid_ms(other) == grid_ms), - ) + let on_grid = queries_of_context( + evaluation_grid_ms(entry), + reaching_shift.get(&shift).into_iter().flatten().copied(), + entries, + ); + anchor_is_evaluation_time(at, on_grid.iter().map(|&q| &entries[q])) }; let walks = starts .into_iter() @@ -425,6 +421,23 @@ pub fn anchor_is_evaluation_time<'e>( }) } +/// The queries of the execution context on evaluation grid `grid_ms` +/// among `reaching` (the queries reaching a node, by index in `entries`): +/// those evaluated on that grid ([`evaluation_grid_ms`]; 1 ms for a query +/// without an entry). Stage 3 (over the queries reaching a memo group or +/// an anchored shift) and [`computations`] (over the queries whose walk +/// reaches a plan node) pick a context's queries with it. +pub fn queries_of_context( + grid_ms: u64, + reaching: impl IntoIterator, + entries: &[QueryWorkloadEntry], +) -> Vec { + reaching + .into_iter() + .filter(|&query| entries.get(query).map_or(1, evaluation_grid_ms) == grid_ms) + .collect() +} + /// The grid, in ms, every evaluation of `entry`'s query falls on /// (`QueryRecurrence::evaluation_grid_ms` of its range evaluation), the /// grid of its evaluation context; 1 ms when not known, a grid on which @@ -603,9 +616,8 @@ pub fn computations( let mut count: HashMap<_, (u64, f64)> = HashMap::new(); for (id, of_node) in consumers(dag, entries) { let entry = count.entry(id).or_default(); - // The queries reaching it, per context: (evaluation grid, anchor). - let mut queries_by_context: BTreeMap<(u64, Option), BTreeSet> = - BTreeMap::new(); + // The queries reaching it, per anchor. + let mut queries_by_anchor: BTreeMap, BTreeSet> = BTreeMap::new(); for consumer in of_node { let (query, anchor) = match consumer { Consumer::Query(query) => (query, None), @@ -616,19 +628,23 @@ pub fn computations( continue; } }; - let grid_ms = entries.get(query).map_or(1, evaluation_grid_ms); - queries_by_context - .entry((grid_ms, anchor)) - .or_default() - .insert(query); + queries_by_anchor.entry(anchor).or_default().insert(query); } - for ((_, anchor), queries) in queries_by_context { - entry.0 += 1; - entry.1 += context_executions_per_second( - anchor, - queries.iter().filter_map(|&query| entries.get(query)), - non_repeating_query_horizon_seconds, - )?; + // One context per (evaluation grid, anchor). + for (anchor, reaching) in queries_by_anchor { + let grids: BTreeSet = reaching + .iter() + .map(|&query| entries.get(query).map_or(1, evaluation_grid_ms)) + .collect(); + for grid_ms in grids { + let queries = queries_of_context(grid_ms, reaching.iter().copied(), entries); + entry.0 += 1; + entry.1 += context_executions_per_second( + anchor, + queries.iter().filter_map(|&query| entries.get(query)), + non_repeating_query_horizon_seconds, + )?; + } } } Ok(dag diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index a84d97285..24809a696 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -524,3 +524,18 @@ fn an_instant_at_end_input_shares_the_evaluation_context() { ); assert!(objective < 0.04, "objective {objective}"); } + +/// `sum_over_time(m[1m])` every 20 s at phase 10 s and +/// `max_over_time(m[1m])` every 10 s: one 10 s grid, two phases. Stage 3 +/// and `plan_cost` pick the queries of each context with one rule +/// (`queries_of_context`), and the optimum is the plan's cost. +#[test] +fn the_objective_is_the_plan_cost_on_two_phases_of_one_grid() { + let mut late = requested("sum_over_time(m[1m])", 20_000, None); + late["demand"]["fixed_interval_at"]["evaluation_phase"] = serde_json::json!(10_000); + let w = over_m(vec![late, requested("max_over_time(m[1m])", 10_000, None)]); + assert_equal( + "two phases of one grid", + objective_and_plan_cost(&w, &CostInputs::default(), true), + ); +} diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 9012b723f..91e0e9c91 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -53,6 +53,7 @@ use std::collections::HashMap; use asap_cost::plan::execution_demand::{ context_executions_per_second, executions_per_second as rate_of_materialized, + queries_of_context, }; use asap_cost::{ maintained_window_ms, node_resource_usage_and_cost, raw_data_disk_cost_per_second, CostInputs, @@ -71,8 +72,8 @@ use asap_types::stages::{ use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::plan::candidate_node; -use crate::retention::{edge_retention_need_ms, edge_scan_read_ms, output_retention_need_ms}; -use crate::selectable::{evaluation_grid, Execution, SelectableExecutions}; +use crate::retention::{edge_retention_need_ms, edge_scan_read_ms, ms_to_seconds, reads_back}; +use crate::selectable::{Execution, SelectableExecutions}; /// The objective's coefficients /// (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, @@ -106,6 +107,22 @@ pub(crate) struct ExecutionCosts { pub(crate) unbounded_raw_data_on_disk_cost_per_second: HashMap, } +impl ExecutionCosts { + /// $\pi_v h_v$ of candidate `v` retaining `retention_ms`, per second; 0 + /// for a candidate with no retention cost. + pub(crate) fn retention_cost_per_second( + &self, + v: PhysicalCandidateId, + retention_ms: u64, + ) -> f64 { + self.retention_cost_per_retained_second + .get(&v) + .map_or(0.0, |per_retained_second| { + per_retained_second * ms_to_seconds(retention_ms) + }) + } +} + /// The objective's coefficients of every execution and edge of /// `selectable` ([`ExecutionCosts`]; module doc). Errs on nothing: what /// `asap-cost` cannot cost is reported in [`ExecutionCosts::per_execution`] @@ -135,8 +152,8 @@ pub(crate) fn execution_costs( .map(|v| is_query_time_scan(physical, PhysicalCandidateId(v))) .collect(); let producer_of = |position: usize| { - let (_, edge, producer) = selectable.edges()[position]; - (edge, executions[producer].candidate) + let edge = selectable.edges()[position]; + (edge.edge, executions[edge.producer].candidate) }; // Each edge's need, computed once per edge of the physical memo: an // edge is read in every context it is selectable in. @@ -163,16 +180,8 @@ pub(crate) fn execution_costs( *most = most.zip(need_ms).map(|(most, need)| most.max(need)); } }; - for (position, &need_ms) in retention_need_ms_per_edge.iter().enumerate() { - read(producer_of(position).1, need_ms); - } - for (query, answering) in &selectable.answering { - for &index in answering { - read( - executions[index].candidate, - Some(output_retention_need_ms(&entries[query.0])), - ); - } + for (index, _, need_ms) in reads_back(selectable, &retention_need_ms_per_edge, entries) { + read(executions[index].candidate, need_ms); } // How far back a query-time `Scan` reads for the consumer of each edge // out of it, in one run (`None`: read whole), once per memo edge. @@ -205,7 +214,7 @@ pub(crate) fn execution_costs( let Some(read_ms) = *read_ms else { continue; }; - let (_, _, producer) = selectable.edges()[position]; + let producer = selectable.edges()[position].producer; let u = executions[producer].candidate; let most = baseline_scan_read_ms .get_mut(&u) @@ -341,7 +350,7 @@ pub(crate) fn execution_costs( .unwrap_or(1_000) .max(1); let longer = node_retaining(v, Some(retention_ms + delta_ms)); - let delta_seconds = delta_ms as f64 / 1_000.0; + let delta_seconds = ms_to_seconds(delta_ms); let per_retained_second = match is_across_evaluations(physical, v) { true => own_execution.get(&v).and_then(|&index| { let (own, raw_data, per_second) = at_rate[index].as_ref().ok()?; @@ -393,14 +402,17 @@ pub(crate) fn execution_costs( .map(|(execution, at_rate)| { let &(own, raw_data, _) = at_rate.as_ref().map_err(Clone::clone)?; let v = execution.candidate; - Ok(match retention_cost_per_retained_second.get(&v) { + let per_retained_second = retention_cost_per_retained_second.get(&v); + Ok(if query_time_scan[v.0] { // A query-time scan's raw data is charged per source. - _ if query_time_scan[v.0] => own, - Some(per_retained_second) if is_across_evaluations(physical, v) => { - let retention_seconds = most_read_ms[&v].unwrap_or(0) as f64 / 1_000.0; - own + raw_data - per_retained_second * retention_seconds - } - _ => own + raw_data, + own + } else if let (Some(per_retained_second), true) = + (per_retained_second, is_across_evaluations(physical, v)) + { + let retention_seconds = ms_to_seconds(most_read_ms[&v].unwrap_or(0)); + own + raw_data - per_retained_second * retention_seconds + } else { + own + raw_data }) }) .collect(); @@ -408,8 +420,9 @@ pub(crate) fn execution_costs( .edges() .iter() .zip(&scan_read_ms_per_edge) - .map(|(&(consumer, edge, producer), scan_read_ms)| { - let e = physical.edge(edge); + .map(|(edge, scan_read_ms)| { + let (consumer, producer) = (edge.consumer, edge.producer); + let e = physical.edge(edge.edge); let (u, c) = (e.producer_candidate_id, e.consumer_candidate_id); per_execution[producer].as_ref().ok()?; let &(own, raw_data, per_second) = at_rate[consumer].as_ref().ok()?; @@ -499,19 +512,29 @@ pub(crate) fn executions_per_second( memo.groups()[group.0 as usize] .reached_by_queries() .iter() - .filter_map(|&query| entries.get(query)) + .copied() + }; + let entries_of = |queries: Vec| -> Vec<&QueryWorkloadEntry> { + queries + .into_iter() + .filter_map(|query| entries.get(query)) + .collect() }; let rate = match selectable.context(execution.context) { ExecutionContext::Evaluation { grid } => match selectable.request_anchor(execution.context) { Some((at, queries)) => context_executions_per_second( Some(at), - queries.iter().filter_map(|query| entries.get(query.0)), + entries_of(queries.iter().map(|query| query.0).collect()), non_repeating_query_horizon_seconds, ), None => context_executions_per_second( None, - reaching(execution.candidate).filter(|entry| evaluation_grid(entry) == grid), + entries_of(queries_of_context( + grid.period_ms, + reaching(execution.candidate), + entries, + )), non_repeating_query_horizon_seconds, ), }, @@ -520,7 +543,7 @@ pub(crate) fn executions_per_second( rate_of_materialized( properties.timing(), properties.lifetime(), - reaching(m), + entries_of(reaching(m).collect()), non_repeating_query_horizon_seconds, ) } diff --git a/crates/stage3-plan-selection/src/cost/tests.rs b/crates/stage3-plan-selection/src/cost/tests.rs index ba0012f4d..201a7aba8 100644 --- a/crates/stage3-plan-selection/src/cost/tests.rs +++ b/crates/stage3-plan-selection/src/cost/tests.rs @@ -19,7 +19,7 @@ use asap_types::workload::{ use super::*; use crate::retention::edge_retention_need_ms; -use crate::selectable::{QueryId, SelectableExecutions}; +use crate::selectable::{QueryId, SelectableEdge, SelectableExecutions}; const DAY_MS: u64 = 86_400_000; @@ -190,20 +190,14 @@ fn no_execution_reading_a_producer_costs_less_than_nothing() { let least_need_ms = selectable .edges() .iter() - .filter(|&&(_, _, producer)| selectable.executions()[producer].candidate == v) - .filter_map(|&(_, edge, _)| edge_retention_need_ms(&physical, &entries, edge)) + .filter(|edge| selectable.executions()[edge.producer].candidate == v) + .filter_map(|edge| edge_retention_need_ms(&physical, &entries, edge.edge)) .min() .unwrap_or(0); - costs - .retention_cost_per_retained_second - .get(&v) - .copied() - .unwrap_or(0.0) - * least_need_ms as f64 - / 1_000.0 + costs.retention_cost_per_second(v, least_need_ms) }; let mut checked = 0; - for (index, &(consumer, edge, _)) in selectable.edges().iter().enumerate() { + for (index, &SelectableEdge { consumer, edge, .. }) in selectable.edges().iter().enumerate() { let (Ok(execution), Some(edge_cost)) = (&costs.per_execution[consumer], costs.per_edge[index]) else { diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 83941592c..66040b7ac 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -92,7 +92,7 @@ pub fn select_plan( costs.per_execution[index].is_ok() && !must_not_read(physical, query, execution.candidate) }); if selectable - .answering + .answering() .iter() .any(|(_, executions)| executions.is_empty()) { diff --git a/crates/stage3-plan-selection/src/milp/dominance.rs b/crates/stage3-plan-selection/src/milp/dominance.rs new file mode 100644 index 000000000..d5566c008 --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/dominance.rs @@ -0,0 +1,105 @@ +//! Executions left out of the selection MILP because another execution +//! dominates them: one with $\ell$ = None whose `OneEvaluation` twin does +//! all it does at no more cost. + +use std::collections::{BTreeSet, HashMap}; + +use asap_types::ir::properties::ExecutionTiming; +use asap_types::stages::Lifetime; + +use super::model::SelectionInputs; +use crate::selectable::QueryId; + +/// Per execution, by position in `SelectableExecutions::executions`, +/// whether it is a query-time execution with $\ell$ = None that its +/// `OneEvaluation` twin $b$ (the same logical candidate, so the same θ and +/// the same error bounds, in the same context) dominates: $b$ costs no +/// more, reads each of its producers on the same input at no more, is +/// read by each of its consumers on the same input at no more, and answers +/// every query output it answers. A plan selecting it selects $b$ in its +/// place at no more cost and stays feasible: $b$'s (F) is the looser +/// $x \le$ uses, its edges are the same ones (so (A), (B), (E) and (X) +/// hold as before), and (K), per execution, sees the same θ through the +/// same producers. A candidate with $h_v$ or raw data on disk is never +/// left out so, its retention depending on its own readers. +/// +/// Example: a `SummaryAgg` KLL build read by one readout in a context, +/// offered with $\ell$ = None and `OneEvaluation` at equal cost: the None +/// execution is left out. +pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> Vec { + let selectable = inputs.selectable; + let physical = inputs.physical; + let costs = inputs.costs; + let executions = selectable.executions(); + let edges = selectable.edges(); + let count = executions.len(); + let no_more = |cost: f64, than: f64| cost <= than + 1e-12 * than.abs().max(1e-12); + let mut answers: Vec> = vec![BTreeSet::new(); count]; + for (query, answering) in selectable.answering() { + for &index in answering { + answers[index].insert(*query); + } + } + // The query-time executions of each logical candidate in each context, + // without $h_v$ or raw data and costed: (with ℓ = None, OneEvaluation). + let mut twins: HashMap<_, (Vec, Vec)> = HashMap::new(); + for (index, execution) in executions.iter().enumerate() { + let v = execution.candidate; + let candidate = &physical.physical_candidates[v.0]; + let properties = candidate.physical_properties; + if properties.timing() != ExecutionTiming::QueryTime + || costs.retention_cost_per_retained_second.contains_key(&v) + || costs.raw_data_source.contains_key(&v) + || costs.per_execution[index].is_err() + { + continue; + } + let (none, one_evaluation) = twins + .entry((execution.context, candidate.logical_candidate_id)) + .or_default(); + match properties.lifetime() { + Lifetime::None => none.push(index), + Lifetime::OneEvaluation(_) => one_evaluation.push(index), + Lifetime::AcrossEvaluations(_) => {} + } + } + let cost = |index: usize| *costs.per_execution[index].as_ref().expect("costed"); + let input_of = |position: usize| physical.edge(edges[position].edge).input_index_of_consumer; + // Whether `b` reads each producer `a` reads on an input, and is read by + // each consumer that reads `a` on its input, over a costed edge at no + // more. + let edges_no_dearer = |a: usize, b: usize| { + let covered = |position: usize, twins: &[usize], same_end: &dyn Fn(usize) -> bool| { + let Some(edge_cost) = costs.per_edge[position] else { + return true; + }; + twins.iter().any(|&twin| { + same_end(twin) + && costs.per_edge[twin].is_some_and(|twin_cost| no_more(twin_cost, edge_cost)) + }) + }; + (0..selectable.input_count(a)).all(|input| { + selectable.edges_into(a, input).iter().all(|&position| { + covered(position, selectable.edges_into(b, input), &|twin| { + edges[twin].producer == edges[position].producer + }) + }) + }) && selectable.edges_out_of(a).iter().all(|&position| { + covered(position, selectable.edges_out_of(b), &|twin| { + edges[twin].consumer == edges[position].consumer + && input_of(twin) == input_of(position) + }) + }) + }; + let mut dominated = vec![false; count]; + for (none, one_evaluation) in twins.values() { + for &a in none { + dominated[a] = one_evaluation.iter().any(|&b| { + no_more(cost(b), cost(a)) + && edges_no_dearer(a, b) + && answers[a].is_subset(&answers[b]) + }); + } + } + dominated +} diff --git a/crates/stage3-plan-selection/src/milp/exact_only.rs b/crates/stage3-plan-selection/src/milp/exact_only.rs index ab34f582f..a60413329 100644 --- a/crates/stage3-plan-selection/src/milp/exact_only.rs +++ b/crates/stage3-plan-selection/src/milp/exact_only.rs @@ -57,30 +57,38 @@ pub(crate) fn exact_only_variables( ) -> BTreeMap<(QueryId, usize), Variable> { let selectable = inputs.selectable; let count = selectable.executions().len(); - // The selectable edges, as producers per consumer and back. - let mut producers = vec![Vec::new(); count]; - let mut consumers = vec![Vec::new(); count]; - for &position in y.keys() { - let (c, _, u) = selectable.edges()[position]; - producers[c].push(u); - consumers[u].push(c); - } + let edges = selectable.edges(); + // The selectable edges (those with a `y`), producers per consumer and + // back. + let producers = |e: usize| { + (0..selectable.input_count(e)) + .flat_map(move |input| selectable.edges_into(e, input)) + .filter(|position| y.contains_key(position)) + .map(|&position| edges[position].producer) + }; + let consumers = |e: usize| { + selectable + .edges_out_of(e) + .iter() + .filter(|position| y.contains_key(position)) + .map(|&position| edges[position].consumer) + }; let forbidden = |query: QueryId, e: usize| { must_not_read(inputs.physical, query, selectable.executions()[e].candidate) }; - let queries: BTreeSet = selectable.answering.iter().map(|(q, _)| *q).collect(); + let queries: BTreeSet = selectable.answering().iter().map(|(q, _)| *q).collect(); let mut tracked = BTreeMap::new(); for query in queries { let mut reached = vec![false; count]; let mut pending: Vec = selectable - .answering + .answering() .iter() .filter(|(q, _)| *q == query) .flat_map(|(_, executions)| executions.iter().copied()) .collect(); while let Some(e) = pending.pop() { if !std::mem::replace(&mut reached[e], true) { - pending.extend(&producers[e]); + pending.extend(producers(e)); } } let mut reaches_forbidden = vec![false; count]; @@ -89,7 +97,7 @@ pub(crate) fn exact_only_variables( .collect(); while let Some(e) = pending.pop() { if reached[e] && !std::mem::replace(&mut reaches_forbidden[e], true) { - pending.extend(&consumers[e]); + pending.extend(consumers(e)); } } for e in (0..count).filter(|&e| reaches_forbidden[e] && !forbidden(query, e)) { @@ -125,7 +133,8 @@ pub(crate) fn exact_only_constraints( readers.entry(execution).or_default().push((query, a)); } for (&position, &y) in y { - let (consumer, _, producer) = selectable.edges()[position]; + let edge = selectable.edges()[position]; + let (consumer, producer) = (edge.consumer, edge.producer); for &(query, a_consumer) in readers.get(&consumer).into_iter().flatten() { let v = selectable.executions()[producer].candidate; if must_not_read(inputs.physical, query, v) { diff --git a/crates/stage3-plan-selection/src/milp/graph.rs b/crates/stage3-plan-selection/src/milp/graph.rs index db9378d9e..3bbaa2168 100644 --- a/crates/stage3-plan-selection/src/milp/graph.rs +++ b/crates/stage3-plan-selection/src/milp/graph.rs @@ -20,14 +20,25 @@ pub(crate) struct Components { impl Components { pub(crate) fn of(inputs: &SelectionInputs<'_>) -> Self { let selectable = inputs.selectable; + let edges = selectable.edges(); + let costed = |position: &&usize| inputs.costs.per_edge[**position].is_some(); let of = strongly_connected_components( selectable.executions().len(), - selectable - .edges() - .iter() - .zip(&inputs.costs.per_edge) - .filter(|(_, cost)| cost.is_some()) - .map(|(&(consumer, _, producer), _)| (producer, consumer)), + |producer| { + selectable + .edges_out_of(producer) + .iter() + .filter(costed) + .map(|&position| edges[position].consumer) + .collect() + }, + |consumer| { + (0..selectable.input_count(consumer)) + .flat_map(|input| selectable.edges_into(consumer, input)) + .filter(costed) + .map(|&position| edges[position].producer) + .collect() + }, ); let mut count: HashMap = HashMap::new(); for &component in &of { @@ -38,18 +49,14 @@ impl Components { } } -/// Each node's strongly connected component of the graph with `arcs` -/// (Kosaraju), as a component number. +/// Each node's strongly connected component of the graph whose arcs out of +/// a node are `forward(node)` and into it `backward(node)` (Kosaraju), as +/// a component number. fn strongly_connected_components( node_count: usize, - arcs: impl Iterator, + forward: impl Fn(usize) -> Vec, + backward: impl Fn(usize) -> Vec, ) -> Vec { - let mut forward = vec![Vec::new(); node_count]; - let mut backward = vec![Vec::new(); node_count]; - for (from, to) in arcs { - forward[from].push(to); - backward[to].push(from); - } // Nodes by finishing time of a depth-first search over `forward`. let mut finished = Vec::with_capacity(node_count); let mut visited = vec![false; node_count]; @@ -57,12 +64,12 @@ fn strongly_connected_components( if std::mem::replace(&mut visited[start], true) { continue; } - let mut stack = vec![(start, 0)]; - while let Some((node, next)) = stack.pop() { - if let Some(&to) = forward[node].get(next) { - stack.push((node, next + 1)); + let mut stack = vec![(start, forward(start), 0)]; + while let Some((node, out, next)) = stack.pop() { + if let Some(&to) = out.get(next) { + stack.push((node, out, next + 1)); if !std::mem::replace(&mut visited[to], true) { - stack.push((to, 0)); + stack.push((to, forward(to), 0)); } } else { finished.push(node); @@ -77,7 +84,7 @@ fn strongly_connected_components( component[start] = number; let mut stack = vec![start]; while let Some(node) = stack.pop() { - for &from in &backward[node] { + for from in backward(node) { if component[from] == usize::MAX { component[from] = number; stack.push(from); diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index 1a6d91f90..7caa0e268 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -3,10 +3,12 @@ //! sections 3–5): its variables over execution contexts and one function //! per constraint family ([`model`]), the retention amount $h_v$ and //! constraint (R) ([`retention`]), each query's exact-only reads, -//! constraint (X) ([`exact_only`]), the graph algorithm it needs +//! constraint (X) ([`exact_only`]), the executions other executions +//! dominate ([`dominance`]), the graph algorithm it needs //! ([`graph`]), the bound it scales by ([`upper_bound`]), and the accuracy //! extension ([`accuracy`]). mod accuracy; +mod dominance; pub(crate) mod exact_only; mod graph; pub(crate) mod model; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 369573969..0d9cc2dba 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -4,7 +4,7 @@ //! //! | Family | Variables | Constraints | //! |---|---|---| -//! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_constraints`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | +//! | baseline (section 3) over execution contexts (section 5) | $x_{\kappa,v}$ (with $x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$), $y_{\kappa,u,c,i}$, $z_{q,v}$, $t$, $r_s$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) and one consumer per context of a query-time result that is not materialized [`use_constraints`], each source's raw data on disk once [`raw_data_disk_rows`]; (G), (H): $y$ exists only on admissible edges where the producer delivers on the consumer's context | //! | retention (section 5) | $h_v$ | (R) [`super::retention::retention_constraints`] | //! | exact-only reads per query | $a_{q,e}$ | (X) [`super::exact_only::exact_only_constraints`] | //! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ (`AccuracyVariables`) | (K) `accuracy_constraints` | @@ -27,10 +27,11 @@ use asap_types::ir::operator::Source; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; +use super::dominance::dominated_by_one_evaluation_twin; use super::exact_only::{exact_only_constraints, exact_only_variables}; use super::graph::Components; use super::retention::{least_retention_ms, retention_constraints, retention_variables}; -use super::upper_bound::{dearer_than_a_feasible_plan, dominated_by_one_evaluation_twin}; +use super::upper_bound::dearer_than_a_feasible_plan; use crate::cost::ExecutionCosts; use crate::selectable::{has_one_reader_per_context, Execution, QueryId, SelectableExecutions}; @@ -67,8 +68,8 @@ pub(crate) struct SelectionVariables { /// an execution kept ([`retention_variables`]). h: BTreeMap, /// Per raw-data source a query-time scan keeps on disk at a cost: the - /// cost per second of keeping it, at least each selected scan's - /// ([`raw_data_disk_constraints`]). + /// cost per second of keeping it, at least each selected scan's, + /// scaled by the objective's scale ([`raw_data_disk_rows`]). raw_data_on_disk: BTreeMap, /// $a_{q,e}$: query $q$'s selected path reads execution $e$, where /// (X) needs it ([`exact_only_variables`]). @@ -97,13 +98,11 @@ fn selection_variables( .iter() .zip(&inputs.costs.per_edge) .enumerate() - .filter(|(_, (&(consumer, _, producer), cost))| { - cost.is_some() && kept[consumer] && kept[producer] - }) + .filter(|(_, (edge, cost))| cost.is_some() && kept[edge.consumer] && kept[edge.producer]) .map(|(position, _)| (position, problem.add(variable().binary()))) .collect(); let mut z = BTreeMap::new(); - for (query, executions) in &selectable.answering { + for (query, executions) in selectable.answering() { for &execution in executions { z.entry((*query, execution)) .or_insert_with(|| problem.add(variable().binary())); @@ -157,7 +156,7 @@ fn input_satisfaction_constraints( } /// (B) $y_{\kappa,u,c,i} \le x$ of $u$'s execution it reads, except for -/// a producer with one reader per context ([`reads_once`]), whose (F) row +/// a producer with one reader per context ([`has_one_reader`]), whose (F) row /// implies it. fn producer_selection_constraints( inputs: &SelectionInputs<'_>, @@ -167,21 +166,21 @@ fn producer_selection_constraints( .y .iter() .filter_map(|(&position, &y)| { - let (_, _, producer) = inputs.selectable.edges()[position]; - (!reads_once(inputs, producer)).then(|| constraint!(y <= variables.x[producer])) + let producer = inputs.selectable.edges()[position].producer; + (!has_one_reader(inputs, producer)).then(|| constraint!(y <= variables.x[producer])) }) .collect() } /// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x$ of $v$'s /// execution answering $q$, except for one with one reader per context -/// ([`reads_once`]), whose (F) row implies it. +/// ([`has_one_reader`]), whose (F) row implies it. fn query_coverage_constraints( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, ) -> Vec { let mut constraints = Vec::new(); - for (query, executions) in &inputs.selectable.answering { + for (query, executions) in inputs.selectable.answering() { let covered: Expression = executions .iter() .map(|&execution| variables.z[&(*query, execution)]) @@ -189,7 +188,7 @@ fn query_coverage_constraints( constraints.push(constraint!(covered == 1)); } for (&(_, execution), &z) in &variables.z { - if !reads_once(inputs, execution) { + if !has_one_reader(inputs, execution) { constraints.push(constraint!(z <= variables.x[execution])); } } @@ -201,7 +200,7 @@ fn query_coverage_constraints( /// $\sum y + \sum_q z_{q,v} = x_{\kappa,v}$ ([`use_constraints`]), which /// with $y, z \ge 0$ implies (B) and (D) for it, so those rows are left /// out. -fn reads_once(inputs: &SelectionInputs<'_>, index: usize) -> bool { +fn has_one_reader(inputs: &SelectionInputs<'_>, index: usize) -> bool { let v = inputs.selectable.executions()[index].candidate; has_one_reader_per_context(inputs.physical.physical_candidates[v.0].physical_properties) } @@ -218,7 +217,8 @@ fn acyclicity_constraints( .y .iter() .filter_map(|(&position, &y)| { - let (c, _, u) = inputs.selectable.edges()[position]; + let edge = inputs.selectable.edges()[position]; + let (c, u) = (edge.consumer, edge.producer); if components.of[u] != components.of[c] { return None; } @@ -244,7 +244,7 @@ fn use_constraints( uses_of(inputs, variables) .into_iter() .enumerate() - .map(|(index, used)| match reads_once(inputs, index) { + .map(|(index, used)| match has_one_reader(inputs, index) { true => constraint!(used == variables.x[index]), false => constraint!(variables.x[index] <= used), }) @@ -255,32 +255,37 @@ fn use_constraints( /// that keeps the most: $r_s \ge \pi_v h_v$ for each query-time `Scan` $v$ /// of source $s$ with $h_v$, and $r_s \ge d_v x_{\kappa,v}$ for each kept /// execution of one without, $d_v$ its cost per second of the raw data it -/// keeps retaining everything. The objective's scale is in the rows, so -/// $r_s$ is the scaled cost and its objective coefficient is 1. -fn raw_data_disk_constraints( +/// keeps retaining everything: each row as ($r_s$, its coefficient, the +/// variable it multiplies), unscaled. +fn raw_data_disk_rows( inputs: &SelectionInputs<'_>, variables: &SelectionVariables, kept: &[bool], - objective_scale: f64, -) -> Vec { +) -> Vec<(Variable, f64, Variable)> { let costs = inputs.costs; let mut scans: Vec<(&PhysicalCandidateId, &Source)> = costs.raw_data_source.iter().collect(); scans.sort_unstable_by_key(|(v, _)| **v); - let mut constraints = Vec::new(); + let mut executions_of: HashMap> = HashMap::new(); + for (index, execution) in inputs.selectable.executions().iter().enumerate() { + if kept[index] { + executions_of + .entry(execution.candidate) + .or_default() + .push(index); + } + } + let mut rows = Vec::new(); for (v, source) in scans { let r = variables.raw_data_on_disk[source]; if let Some(&h) = variables.h.get(v) { - let per_retained_second = costs.retention_cost_per_retained_second[v]; - constraints.push(constraint!(r >= per_retained_second * objective_scale * h)); + rows.push((r, costs.retention_cost_per_retained_second[v], h)); } else if let Some(&d) = costs.unbounded_raw_data_on_disk_cost_per_second.get(v) { - for (index, execution) in inputs.selectable.executions().iter().enumerate() { - if execution.candidate == *v && kept[index] { - constraints.push(constraint!(r >= d * objective_scale * variables.x[index])); - } + for &index in executions_of.get(v).into_iter().flatten() { + rows.push((r, d, variables.x[index])); } } } - constraints + rows } /// Per execution, its selected edges to consumers and query outputs: @@ -288,8 +293,7 @@ fn raw_data_disk_constraints( fn uses_of(inputs: &SelectionInputs<'_>, variables: &SelectionVariables) -> Vec { let mut uses: Vec = vec![Expression::default(); variables.x.len()]; for (&position, &y) in &variables.y { - let (_, _, producer) = inputs.selectable.edges()[position]; - uses[producer] += y; + uses[inputs.selectable.edges()[position].producer] += y; } for (&(_, execution), &z) in &variables.z { uses[execution] += z; @@ -337,7 +341,7 @@ pub(crate) enum SelectionMilpError { /// \rho_\kappa c_{\kappa,v} x_{\kappa,v} + \sum \rho_\kappa c_{\kappa,u,c,i} /// y_{\kappa,u,c,i} + \sum_v \pi_v h_v + \sum_s r_s$ ($\pi_v h_v$ of a /// query-time scan is in $r_s$) under (A)–(F), [`use_constraints`], -/// [`raw_data_disk_constraints`], (R) and (X), with HiGHS at gap 0 on one +/// [`raw_data_disk_rows`], (R) and (X), with HiGHS at gap 0 on one /// thread. The executions dearer than a feasible plan /// ([`dearer_than_a_feasible_plan`]) or dominated by their `OneEvaluation` /// twin ([`dominated_by_one_evaluation_twin`]) are left out first, with @@ -381,31 +385,12 @@ pub(crate) fn solve_selection( .map(|(v, &h)| (costs.retention_cost_per_retained_second[v], h)), ) .collect(); - // Scaled by the largest coefficient kept, the raw-data bounds of the - // scans kept included. - let raw_data_bounds = - costs - .raw_data_source - .keys() - .filter_map(|v| match variables.h.contains_key(v) { - true => Some(costs.retention_cost_per_retained_second[v]), - false => costs - .unbounded_raw_data_on_disk_cost_per_second - .get(v) - .copied() - .filter(|_| { - inputs - .selectable - .executions() - .iter() - .zip(&kept) - .any(|(execution, &kept)| kept && execution.candidate == *v) - }), - }); + // Scaled by the largest coefficient kept, the raw-data rows' included. + let raw_data_rows = raw_data_disk_rows(inputs, &variables, &kept); let largest = terms .iter() .map(|(cost, _)| *cost) - .chain(raw_data_bounds) + .chain(raw_data_rows.iter().map(|&(_, coefficient, _)| coefficient)) .fold(0.0_f64, |largest, cost| largest.max(cost.abs())); let scale = match largest > 0.0 && largest.is_finite() { true => 1_000.0 / largest, @@ -417,14 +402,19 @@ pub(crate) fn solve_selection( query_coverage_constraints(inputs, &variables), acyclicity_constraints(inputs, &variables, &components), use_constraints(inputs, &variables), - raw_data_disk_constraints(inputs, &variables, &kept, scale), + // The objective's scale is in the raw-data rows, so $r_s$ is the + // scaled cost and its objective coefficient is 1. + raw_data_rows + .iter() + .map(|&(r, coefficient, variable)| constraint!(r >= coefficient * scale * variable)) + .collect(), retention_constraints(inputs, &variables.h, &variables.y, &variables.z), exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), ] .into_iter() .flatten() .collect(); - // The raw data on disk is scaled by its rows ([`raw_data_disk_constraints`]). + // The raw data on disk is scaled by its rows ([`raw_data_disk_rows`]). let objective: Expression = terms .iter() .map(|&(cost, v)| cost * scale * v) @@ -487,7 +477,7 @@ pub(crate) fn solve_selection( .map(|(&position, _)| position) .collect(); let answering: Vec<(QueryId, usize)> = selectable - .answering + .answering() .iter() .map(|(query, answering)| { let execution = answering @@ -505,8 +495,9 @@ pub(crate) fn solve_selection( edges: selected_edges .iter() .map(|&position| { - let (consumer, edge, producer) = selectable.edges()[position]; - (executions[consumer], edge, executions[producer]) + let edge = selectable.edges()[position]; + let (consumer, producer) = (edge.consumer, edge.producer); + (executions[consumer], edge.edge, executions[producer]) }) .collect(), outputs: answering @@ -540,7 +531,7 @@ pub(crate) fn selection_cost_per_second( let mut raw_data_on_disk: BTreeMap<&Source, f64> = BTreeMap::new(); let mut retention_cost_per_second = 0.0; for (&v, &ms) in &retention_ms { - let cost = costs.retention_cost_per_retained_second[&v] * ms as f64 / 1_000.0; + let cost = costs.retention_cost_per_second(v, ms); match costs.raw_data_source.get(&v) { Some(source) => { let kept = raw_data_on_disk.entry(source).or_default(); diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index ab111bf71..46c9a502b 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -31,7 +31,7 @@ use crate::cost::ExecutionCosts; use crate::milp::exact_only::must_not_read; use crate::plan::plan_of; use crate::retention::edge_retention_need_ms; -use crate::selectable::output_groups; +use crate::selectable::{output_groups, SelectableEdge}; const NONE: Lifetime = Lifetime::None; const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); @@ -155,7 +155,7 @@ fn selection_inputs( .iter() .map(|&(v, per_retained_second)| (PhysicalCandidateId(v), per_retained_second)) .collect(); - let edge_of = |&(_, id, _): &(usize, PhysicalEdgeId, usize)| physical.edge(id); + let edge_of = |edge: &SelectableEdge| physical.edge(edge.edge); let costs = ExecutionCosts { per_execution: selectable .executions() @@ -183,7 +183,7 @@ fn selection_inputs( let edge = edge_of(selectable_edge); retention_cost_per_retained_second .contains_key(&edge.producer_candidate_id) - .then(|| edge_retention_need_ms(physical, entries, selectable_edge.1)) + .then(|| edge_retention_need_ms(physical, entries, selectable_edge.edge)) .flatten() }) .collect(), @@ -642,7 +642,7 @@ fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { ]; let entries = [every(10_000), every(30_000)]; let (mut selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); - let q0_context = selectable.executions()[selectable.answering[0].1[0]].context; + let q0_context = selectable.executions()[selectable.answering()[0].1[0]].context; for (index, execution) in selectable.executions().iter().enumerate() { if execution.candidate.0 == exact_quantile && execution.context == q0_context { costs.per_execution[index] = Err("not costable".into()); @@ -726,7 +726,7 @@ fn a_none_execution_its_one_evaluation_twin_dominates_is_left_out() { let entries = every_10s(1); let dominated = |w: &[f64], twin_edge_costed: bool| { let (selectable, mut costs) = selection_inputs(&physical, &entries, w, &[], &[]); - for (position, &(_, edge, _)) in selectable.edges().iter().enumerate() { + for (position, &SelectableEdge { edge, .. }) in selectable.edges().iter().enumerate() { if physical.edge(edge).producer_candidate_id.0 == 1 && !twin_edge_costed { costs.per_edge[position] = None; } diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs index a7cd0b41d..23ac8386e 100644 --- a/crates/stage3-plan-selection/src/milp/retention.rs +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -4,14 +4,14 @@ //! executions $x_{\kappa,v}$ and edges $y_{\kappa,u,c,i}$ it selects over //! are [`crate::selectable::SelectableExecutions`]'s, in [`super::model`]. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; use asap_types::stages::PhysicalCandidateId; use super::model::SelectionInputs; -use crate::retention::output_retention_need_ms; +use crate::retention::{ms_to_seconds, reads_back, Reader}; use crate::selectable::QueryId; /// $h_v$ in seconds retained (the unit keeps (R)'s coefficients near the @@ -58,19 +58,21 @@ pub(crate) fn retention_constraints( ) -> Vec { let executions = inputs.selectable.executions(); let mut constraints = Vec::new(); - for (&position, &y) in y { - let (_, _, producer) = inputs.selectable.edges()[position]; - let Some(&h) = h.get(&executions[producer].candidate) else { + for (index, reader, need_ms) in reads_back( + inputs.selectable, + &inputs.costs.retention_need_ms_per_edge, + inputs.entries, + ) { + let Some(&h) = h.get(&executions[index].candidate) else { continue; }; - let need_ms = inputs.costs.retention_need_ms_per_edge[position] - .expect("a candidate with h_v is read within a bound"); - constraints.push(constraint!(h >= seconds(need_ms) * y)); - } - for (&(query, execution), &z) in z { - if let Some(&h) = h.get(&executions[execution].candidate) { - let need_ms = output_retention_need_ms(&inputs.entries[query.0]); - constraints.push(constraint!(h >= seconds(need_ms) * z)); + let read = match reader { + Reader::Edge(position) => y.get(&position), + Reader::Output(query) => z.get(&(query, index)), + }; + if let Some(&read) = read { + let need_ms = need_ms.expect("a candidate with h_v is read within a bound"); + constraints.push(constraint!(h >= ms_to_seconds(need_ms) * read)); } } constraints @@ -88,30 +90,23 @@ pub(crate) fn least_retention_ms( ) -> HashMap { let executions = inputs.selectable.executions(); let retaining = &inputs.costs.retention_cost_per_retained_second; - let reads = edges - .into_iter() - .filter_map(|position| { - let (_, _, producer) = inputs.selectable.edges()[position]; - Some(( - executions[producer].candidate, - inputs.costs.retention_need_ms_per_edge[position]?, - )) - }) - .chain(outputs.into_iter().map(|(query, execution)| { - ( - executions[execution].candidate, - output_retention_need_ms(&inputs.entries[query.0]), - ) - })); + let edges: HashSet = edges.into_iter().collect(); + let outputs: HashSet<(QueryId, usize)> = outputs.into_iter().collect(); let mut retention_ms: HashMap = HashMap::new(); - for (v, need_ms) in reads.filter(|(v, _)| retaining.contains_key(v)) { - let most = retention_ms.entry(v).or_default(); - *most = (*most).max(need_ms); + for (index, reader, need_ms) in reads_back( + inputs.selectable, + &inputs.costs.retention_need_ms_per_edge, + inputs.entries, + ) { + let selected = match reader { + Reader::Edge(position) => edges.contains(&position), + Reader::Output(query) => outputs.contains(&(query, index)), + }; + let v = executions[index].candidate; + if let (true, Some(need_ms), true) = (selected, need_ms, retaining.contains_key(&v)) { + let most = retention_ms.entry(v).or_default(); + *most = (*most).max(need_ms); + } } retention_ms } - -/// `ms` in seconds, the unit of $h_v$ in the MILP. -fn seconds(ms: u64) -> f64 { - ms as f64 / 1_000.0 -} diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index 5fe4d6d27..c363f9aa9 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -11,28 +11,18 @@ //! plans became 4e-7, and the solver returned a plan 12.6% dearer. The //! greedy plan bounds the optimum by about 0.0088/s, so the chain is left //! out and the largest coefficient left is about 0.006/s. -//! -//! Also left out: an execution with $\ell$ = None that its `OneEvaluation` -//! twin dominates ([`dominated_by_one_evaluation_twin`]). - -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::{CandidateForMemoGroupId, ExecutionContextId, Lifetime}; +use std::collections::{BTreeMap, HashMap}; use super::exact_only::must_not_read; use super::model::{selection_cost_per_second, SelectionInputs}; -use crate::retention::output_retention_need_ms; +use crate::retention::{reads_back, Reader}; use crate::selectable::{has_one_reader_per_context, QueryId}; /// How many times the greedy plan is rebuilt without an execution that /// made it infeasible before giving up on a bound. const REPAIRS: usize = 32; -/// Per (context, logical candidate), its query-time executions as -/// (execution, lifetime, cost per second). -type Twins = HashMap<(ExecutionContextId, CandidateForMemoGroupId), Vec<(usize, Lifetime, f64)>>; - /// Per input of an execution, the edge its cheapest tree reads it over, as /// (edge position, producer execution). type TreeInputs = Vec>; @@ -58,33 +48,25 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec> = vec![None; count]; - let mut read = |index: usize, need_ms: u64| { - let least = &mut least_need_ms[index]; - *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); - }; - for (position, &(_, _, producer)) in selectable.edges().iter().enumerate() { - if let (Some(_), Some(need_ms)) = ( - costs.per_edge[position], - costs.retention_need_ms_per_edge[position], - ) { - read(producer, need_ms); - } - } - for (query, answering) in &selectable.answering { - for &index in answering { - read(index, output_retention_need_ms(&inputs.entries[query.0])); + for (index, reader, need_ms) in reads_back( + selectable, + &costs.retention_need_ms_per_edge, + inputs.entries, + ) { + let costed = match reader { + Reader::Edge(position) => costs.per_edge[position].is_some(), + Reader::Output(_) => true, + }; + if let (true, Some(need_ms)) = (costed, need_ms) { + let least = &mut least_need_ms[index]; + *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); } } let least_retention_cost = |index: usize| { let v = executions[index].candidate; - match ( - costs.retention_cost_per_retained_second.get(&v), - least_need_ms[index], - ) { - (Some(per_retained_second), Some(need_ms)) - if !costs.raw_data_source.contains_key(&v) => - { - per_retained_second * need_ms as f64 / 1_000.0 + match least_need_ms[index] { + Some(need_ms) if !costs.raw_data_source.contains_key(&v) => { + costs.retention_cost_per_second(v, need_ms) } _ => 0.0, } @@ -140,7 +122,7 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { // Per query, the executions its trees leave out: those it must not // read, and those a repair left out for it. let mut excluded: BTreeMap> = BTreeMap::new(); - for (query, _) in &selectable.answering { + for (query, _) in selectable.answering() { excluded .entry(*query) .or_insert_with(|| (0..count).map(|e| forbidden(*query, e)).collect()); @@ -150,7 +132,7 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { 'repair: for _ in 0..REPAIRS { let mut plan = GreedyPlan::new(count); let mut answering: Vec<(QueryId, usize)> = Vec::new(); - for (query, outputs) in &selectable.answering { + for (query, outputs) in selectable.answering() { let left_out = &excluded[query]; let (tree_cost, chosen) = trees .entry(left_out.clone()) @@ -202,7 +184,7 @@ fn cheapest_trees(inputs: &SelectionInputs<'_>, excluded: &[bool]) -> (Vec, .iter() .filter_map(|&position| { let c = inputs.costs.per_edge[position]?; - let (_, _, u) = selectable.edges()[position]; + let u = selectable.edges()[position].producer; (!excluded[u] && tree_cost[u].is_finite()) .then(|| (position, u, c + tree_cost[u])) }) @@ -323,91 +305,3 @@ impl GreedyPlan { Ok(()) } } - -/// Per execution, by position in `SelectableExecutions::executions`, -/// whether it is a query-time execution with $\ell$ = None that its -/// `OneEvaluation` twin $b$ (the same logical candidate, so the same θ and -/// the same error bounds, in the same context) dominates: $b$ costs no -/// more, reads each of its producers on the same input at no more, is -/// read by each of its consumers on the same input at no more, and answers -/// every query output it answers. A plan selecting it selects $b$ in its -/// place at no more cost and stays feasible: $b$'s (F) is the looser -/// $x \le$ uses, its edges are the same ones (so (A), (B), (E) and (X) -/// hold as before), and (K), per execution, sees the same θ through the -/// same producers. A candidate with $h_v$ or raw data on disk is never -/// left out so, its retention depending on its own readers. -/// -/// Example: a `SummaryAgg` KLL build read by one readout in a context, -/// offered with $\ell$ = None and `OneEvaluation` at equal cost: the None -/// execution is left out. -pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> Vec { - let selectable = inputs.selectable; - let physical = inputs.physical; - let costs = inputs.costs; - let executions = selectable.executions(); - let count = executions.len(); - let no_more = |cost: f64, than: f64| cost <= than + 1e-12 * than.abs().max(1e-12); - // Per execution, its costed edges in, by (producer, input), and out, by - // (consumer, input). - let mut edges_in: Vec> = vec![HashMap::new(); count]; - let mut edges_out: Vec> = vec![HashMap::new(); count]; - for (position, &(consumer, edge, producer)) in selectable.edges().iter().enumerate() { - let Some(cost) = costs.per_edge[position] else { - continue; - }; - let input = physical.edge(edge).input_index_of_consumer; - edges_in[consumer].insert((producer, input), cost); - edges_out[producer].insert((consumer, input), cost); - } - let mut answers: Vec> = vec![BTreeSet::new(); count]; - for (query, answering) in &selectable.answering { - for &index in answering { - answers[index].insert(*query); - } - } - // The query-time executions of each logical candidate in each context, - // without $h_v$ or raw data, with their cost. - let mut twins: Twins = HashMap::new(); - for (index, execution) in executions.iter().enumerate() { - let v = execution.candidate; - let candidate = &physical.physical_candidates[v.0]; - let properties = candidate.physical_properties; - if properties.timing() != ExecutionTiming::QueryTime - || costs.retention_cost_per_retained_second.contains_key(&v) - || costs.raw_data_source.contains_key(&v) - { - continue; - } - let Ok(cost) = costs.per_execution[index] else { - continue; - }; - twins - .entry((execution.context, candidate.logical_candidate_id)) - .or_default() - .push((index, properties.lifetime(), cost)); - } - let mut dominated = vec![false; count]; - for members in twins.values() { - for &(a, lifetime, cost_a) in members { - if lifetime != Lifetime::None { - continue; - } - dominated[a] = members.iter().any(|&(b, lifetime, cost_b)| { - matches!(lifetime, Lifetime::OneEvaluation(_)) - && no_more(cost_b, cost_a) - && edges_in[a].iter().all(|(key, &cost)| { - edges_in[b] - .get(key) - .is_some_and(|&twin| no_more(twin, cost)) - }) - && edges_out[a].iter().all(|(key, &cost)| { - edges_out[b] - .get(key) - .is_some_and(|&twin| no_more(twin, cost)) - }) - && answers[a].is_subset(&answers[b]) - }); - } - } - dominated -} diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 35e0d5332..40f7582e3 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -3,7 +3,8 @@ //! over one edge, $\operatorname{need}_{\kappa,u,c,i}$ of constraint (R), //! or for a query reading it directly, and how far back one execution of a //! query-time `Scan` reads for that reader: [`asap_cost::read_back_ms`] -//! over candidates +//! over candidates; and every read of an execution's rows +//! ([`reads_back`]) //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 5, "Cost and retention"). @@ -14,6 +15,8 @@ use asap_types::stages::{ }; use asap_types::workload::QueryWorkloadEntry; +use crate::selectable::{QueryId, SelectableExecutions}; + /// $\operatorname{need}_{\kappa,u,c,i}$: how far back from the evaluation /// time `edge`'s producer $u$ must keep its rows for its consumer $c$, with /// $c$ $u$'s only reader and every candidate that may read $c$ above it @@ -52,10 +55,56 @@ pub(crate) fn edge_scan_read_ms( /// How far back a query reads the output it reads directly: its /// range-evaluation span (0 without one). -pub(crate) fn output_retention_need_ms(entry: &QueryWorkloadEntry) -> u64 { +fn output_retention_need_ms(entry: &QueryWorkloadEntry) -> u64 { entry.range_evaluation.map_or(0, |range| range.span_ms) } +/// What reads an execution's rows back ([`reads_back`]): a selectable edge +/// out of it, by position in [`SelectableExecutions::edges`], or a query +/// output it may answer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Reader { + Edge(usize), + Output(QueryId), +} + +/// Every read of an execution's rows, as (the execution read, by position +/// in [`SelectableExecutions::executions`]; its reader; how far back it +/// reads, in ms, `None` when unbounded): each selectable edge into its +/// producer, `need_ms_per_edge` by edge position +/// ([`edge_retention_need_ms`]), and each query output into each execution +/// that may answer it, the query's range-evaluation span. Costing ($h_v$ +/// at its largest), (R), the least $h_v$ of a selection and the greedy +/// bound each fold it. +pub(crate) fn reads_back<'a>( + selectable: &'a SelectableExecutions, + need_ms_per_edge: &'a [Option], + entries: &'a [QueryWorkloadEntry], +) -> impl Iterator)> + 'a { + let edges = selectable + .edges() + .iter() + .zip(need_ms_per_edge) + .enumerate() + .map(|(position, (edge, &need_ms))| (edge.producer, Reader::Edge(position), need_ms)); + let outputs = selectable + .answering() + .iter() + .flat_map(move |(query, answering)| { + let need_ms = output_retention_need_ms(&entries[query.0]); + answering + .iter() + .map(move |&index| (index, Reader::Output(*query), Some(need_ms))) + }); + edges.chain(outputs) +} + +/// `ms` in seconds, the unit of $h_v$ in the MILP and of the costs per +/// second retained. +pub(crate) fn ms_to_seconds(ms: u64) -> f64 { + ms as f64 / 1_000.0 +} + /// [`asap_cost::read_back_ms`] of `edge`'s producer over candidates, read /// by `edge`'s consumer only and above it by every candidate that may read /// one (its admissible edges out), each read as its operator at its timing diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index 22caea1cd..99b1d9574 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -29,10 +29,10 @@ //! edge, only where the producer delivers on $\kappa$ //! (`PhysicalCandidate::delivers_on`). -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use asap_cost::plan::execution_demand::{ - anchor_is_evaluation_time, evaluation_grid_ms, request_time_grid_ms, + anchor_is_evaluation_time, evaluation_grid_ms, queries_of_context, request_time_grid_ms, }; use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; @@ -67,6 +67,22 @@ pub(crate) struct Execution { pub(crate) candidate: PhysicalCandidateId, } +/// One $y_{\kappa,u,c,i}$ the selection may choose: edge `edge` from the +/// producer execution `producer` into the consumer execution `consumer`, +/// executions by position in [`SelectableExecutions::executions`]; +/// $\kappa$ is the consumer's context. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct SelectableEdge { + pub(crate) consumer: usize, + pub(crate) edge: PhysicalEdgeId, + pub(crate) producer: usize, +} + +/// A context's request anchor ([`SelectableExecutions::request_anchor`]): +/// the anchor of the `TimeShift` whose inputs it runs, and the queries +/// whose requests it runs for. +type RequestAnchor = Option<(AtModifier, Vec)>; + /// The distinct contexts of a workload and every execution and edge the /// selection may choose, with the executions that may answer each query /// output ([`Self::new`]). Executions are referred to by position in @@ -79,21 +95,24 @@ pub(crate) struct SelectableExecutions { /// the queries whose requests they run for (those on the shift's /// evaluation grid that reach it), ascending; `None` for any other /// context. - request_anchor_of_context: Vec)>>, + request_anchor_of_context: Vec, + /// The id of each context with its request anchor. + context_ids: HashMap<(ExecutionContext, RequestAnchor), ExecutionContextId>, /// Every execution reached from a query output, ascending. executions: Vec, - /// Every $y_{\kappa,u,c,i}$ as (consumer execution, edge $(u,c,i)$, - /// producer execution), executions by position, ascending; $\kappa$ is - /// the consumer's context. - edges: Vec<(usize, PhysicalEdgeId, usize)>, + /// Every $y_{\kappa,u,c,i}$, ascending. + edges: Vec, /// Per execution, by position, and per input of its candidate, the /// positions in [`Self::edges`] of the edges into it. edges_into: Vec>>, + /// Per execution, by position, the positions in [`Self::edges`] of the + /// edges out of it. + edges_out_of: Vec>, /// Each query output reference (a query, and the group its operator /// root, or one operator of its scalar root, reads), in /// [`output_groups`] order, with the executions, by position, that may /// answer it: $z_{q,v}$ exists only for them. - pub(crate) answering: Vec<(QueryId, Vec)>, + answering: Vec<(QueryId, Vec)>, } impl SelectableExecutions { @@ -112,9 +131,11 @@ impl SelectableExecutions { let mut selectable = Self { contexts: Vec::new(), request_anchor_of_context: Vec::new(), + context_ids: HashMap::new(), executions: Vec::new(), edges: Vec::new(), edges_into: Vec::new(), + edges_out_of: Vec::new(), answering: Vec::new(), }; let mut reached: BTreeSet = BTreeSet::new(); @@ -173,7 +194,11 @@ impl SelectableExecutions { }; selectable.edges = edges .into_iter() - .map(|(consumer, edge, producer)| (position(consumer), edge, position(producer))) + .map(|(consumer, edge, producer)| SelectableEdge { + consumer: position(consumer), + edge, + producer: position(producer), + }) .collect(); selectable.answering = answering .into_iter() @@ -184,9 +209,11 @@ impl SelectableExecutions { .iter() .map(|execution| vec![Vec::new(); physical.input_count(execution.candidate)]) .collect(); - for (position, &(consumer, edge, _)) in selectable.edges.iter().enumerate() { - selectable.edges_into[consumer][physical.edge(edge).input_index_of_consumer] + selectable.edges_out_of = vec![Vec::new(); selectable.executions.len()]; + for (position, edge) in selectable.edges.iter().enumerate() { + selectable.edges_into[edge.consumer][physical.edge(edge.edge).input_index_of_consumer] .push(position); + selectable.edges_out_of[edge.producer].push(position); } selectable } @@ -196,21 +223,17 @@ impl SelectableExecutions { fn context_id( &mut self, context: ExecutionContext, - request_anchor: Option<(AtModifier, Vec)>, + request_anchor: RequestAnchor, ) -> ExecutionContextId { - match self - .contexts - .iter() - .zip(&self.request_anchor_of_context) - .position(|known| known == (&context, &request_anchor)) - { - Some(index) => ExecutionContextId(index), - None => { + let id = ExecutionContextId(self.contexts.len()); + *self + .context_ids + .entry((context, request_anchor.clone())) + .or_insert_with(|| { self.contexts.push(context); self.request_anchor_of_context.push(request_anchor); - ExecutionContextId(self.contexts.len() - 1) - } - } + id + }) } /// The execution of producer `u` a consumer whose inputs run in @@ -283,16 +306,17 @@ impl SelectableExecutions { else { return consumer.context; }; - let queries: Vec = memo.groups()[group.0 as usize] - .reached_by_queries() - .iter() - .filter(|&&query| { - entries - .get(query) - .is_some_and(|entry| evaluation_grid(entry) == grid) - }) - .map(|&query| QueryId(query)) - .collect(); + let queries: Vec = queries_of_context( + grid.period_ms, + memo.groups()[group.0 as usize] + .reached_by_queries() + .iter() + .copied(), + entries, + ) + .into_iter() + .map(QueryId) + .collect(); if anchor_is_evaluation_time(at, queries.iter().map(|query| &entries[query.0])) { return consumer.context; } @@ -348,12 +372,23 @@ impl SelectableExecutions { &self.executions } - /// Every $y_{\kappa,u,c,i}$ as (consumer execution, edge, producer - /// execution), executions by position, ascending. - pub(crate) fn edges(&self) -> &[(usize, PhysicalEdgeId, usize)] { + /// Every $y_{\kappa,u,c,i}$, ascending. + pub(crate) fn edges(&self) -> &[SelectableEdge] { &self.edges } + /// The positions in [`Self::edges`] of the edges out of the execution + /// at position `producer`. + pub(crate) fn edges_out_of(&self, producer: usize) -> &[usize] { + &self.edges_out_of[producer] + } + + /// Each query output reference with the executions, by position, that + /// may answer it, in [`output_groups`] order. + pub(crate) fn answering(&self) -> &[(QueryId, Vec)] { + &self.answering + } + /// The positions in [`Self::edges`] of the edges into input `input` of /// the execution at position `consumer`. pub(crate) fn edges_into(&self, consumer: usize, input: usize) -> &[usize] { diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index d0c17c376..f4239de2e 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -64,14 +64,13 @@ impl PhysicalCandidate { Lifetime::AcrossEvaluations(_) => { let logical = &candidates.logical_candidates; let group = self.logical_candidate_id.group; - let chain_tumbling_window_width_ms = - self.chain_tumbling_window_width_ms(candidates); let group_properties = &logical.memo.groups()[group.0 as usize].group_properties; let final_at_every_time = !group_properties.depends_on_evaluation_time && (self.physical_properties.timing() == ExecutionTiming::IngestionTime || logical.memo.data_arrival() == DataArrival::AtRest || group_properties.scanned_sources.is_empty()); - if chain_tumbling_window_width_ms.is_none() && final_at_every_time { + if self.chain_tumbling_window_width_ms(candidates).is_none() && final_at_every_time + { return true; } let (period_ms, phase_ms) = match context { @@ -86,17 +85,11 @@ impl PhysicalCandidate { } } }; - match chain_tumbling_window_width_ms { - Some(width_ms) => { - period_ms.is_multiple_of(width_ms) && phase_ms.is_multiple_of(width_ms) - } - None => { - logical.evaluation_grid_of_group[group.0 as usize].is_some_and(|grid| { - period_ms.is_multiple_of(grid.period_ms) - && phase_ms % grid.period_ms == grid.phase_ms % grid.period_ms - }) - } - } + self.own_schedule(candidates).is_some_and(|stored| { + stored.width_ms > 0 + && period_ms.is_multiple_of(stored.width_ms) + && phase_ms % stored.width_ms == stored.phase_ms % stored.width_ms + }) } } } From b83abbf73a67dc9b87dc988c3051c3712dbfddae Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:38:50 +0000 Subject: [PATCH 13/23] perf(stage3): h_v is at least the least need over an execution's uses times x (#1207 PR 7) --- .../stage3-plan-selection/src/milp/model/mod.rs | 8 +++++++- .../stage3-plan-selection/src/milp/retention.rs | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 0d9cc2dba..221d30603 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -408,7 +408,13 @@ pub(crate) fn solve_selection( .iter() .map(|&(r, coefficient, variable)| constraint!(r >= coefficient * scale * variable)) .collect(), - retention_constraints(inputs, &variables.h, &variables.y, &variables.z), + retention_constraints( + inputs, + &variables.h, + &variables.x, + &variables.y, + &variables.z, + ), exact_only_constraints(inputs, &variables.read_by_query, &variables.y, &variables.z), ] .into_iter() diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs index 23ac8386e..92820c218 100644 --- a/crates/stage3-plan-selection/src/milp/retention.rs +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -50,14 +50,23 @@ pub(crate) fn retention_variables( /// query reading it directly (its range-evaluation span), a constant per /// edge, so (R) is linear. Query latency is ignored: a result counts as /// read when its evaluation starts. +/// +/// With them, per execution $u$ of such a $v$, $h_v \ge +/// \min_{\text{uses}} \operatorname{need} \cdot x_u$ over the reads that +/// may select it: (F) gives a selected execution a use, and (R) then +/// retains at least the least need. It is implied for integer $x$, $y$, +/// $z$, but tightens the relaxation where fractional $y$ leave $h_v$ low. pub(crate) fn retention_constraints( inputs: &SelectionInputs<'_>, h: &BTreeMap, + x: &[Variable], y: &BTreeMap, z: &BTreeMap<(QueryId, usize), Variable>, ) -> Vec { let executions = inputs.selectable.executions(); let mut constraints = Vec::new(); + // The least need over each execution's uses, for one with $h_v$. + let mut least_need_ms: BTreeMap = BTreeMap::new(); for (index, reader, need_ms) in reads_back( inputs.selectable, &inputs.costs.retention_need_ms_per_edge, @@ -73,6 +82,14 @@ pub(crate) fn retention_constraints( if let Some(&read) = read { let need_ms = need_ms.expect("a candidate with h_v is read within a bound"); constraints.push(constraint!(h >= ms_to_seconds(need_ms) * read)); + let least = least_need_ms.entry(index).or_insert(need_ms); + *least = (*least).min(need_ms); + } + } + for (index, need_ms) in least_need_ms { + if need_ms > 0 { + let h = h[&executions[index].candidate]; + constraints.push(constraint!(h >= ms_to_seconds(need_ms) * x[index])); } } constraints From 1c758831f4274e1298f668bbd09cc0208b17843e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:12:02 +0000 Subject: [PATCH 14/23] fix(stage3,cost): rate a context by the queries whose walk reaches it; @ start() per span (#1207 PR 7) - `asap_cost::plan::execution_demand::query_reads`: how each query reads the nodes below it, per (query, anchor), over a plan or a memo. `consumers` (plan_cost) walks plan nodes with it, Stage 3 memo groups (`SelectableExecutions::readers_of`). A run in an evaluation context is rated by the queries reading its group in their evaluation context, not by every query reaching the group: an instant `m[5m] @ end()` beside a range `m[5m] @ end() offset 10m` cost 0.3642/s in the MILP for a plan `plan_cost` charged 0.1542/s. Stage 3 walks every candidate of a group, `plan_cost` the selected plan; the difference is documented. - `@ start()`: `evaluation_rate` of the request schedules per span, summed over spans; `anchored_request_recurrence` deleted. - `ExecutionCosts::least_need_ms_per_execution`, computed once, read by the greedy bound and the h_v >= need * x rows. - The bound uses `has_one_reader`; the query-time `Scan` reads `raw_rows_back`; `Candidates` holds its edge; `SelectedEdge`; `is_across_evaluations` wrapper inlined; module docs and planner-layering.md list dominance.rs. --- crates/cost/src/node/usage/mod.rs | 19 +- crates/cost/src/plan/execution_demand/mod.rs | 289 ++++++++++-------- .../cost/src/plan/execution_demand/tests.rs | 16 +- .../tests/stage3_objective.rs | 38 +++ crates/stage3-plan-selection/src/cost.rs | 63 ++-- crates/stage3-plan-selection/src/milp/mod.rs | 6 +- .../src/milp/model/mod.rs | 26 +- .../src/milp/model/tests.rs | 13 +- .../src/milp/retention.rs | 20 +- .../src/milp/upper_bound.rs | 37 +-- crates/stage3-plan-selection/src/plan.rs | 9 +- crates/stage3-plan-selection/src/retention.rs | 65 ++-- .../stage3-plan-selection/src/selectable.rs | 76 ++++- .../design_docs/proposals/planner-layering.md | 2 +- 14 files changed, 425 insertions(+), 254 deletions(-) diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index c758624b2..6411a13ee 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -371,9 +371,10 @@ pub fn node_resource_usage_and_cost<'a>( .sum(); (by_rate, rows_held) }; - // The raw rows its sources keep back `lookback_ms` (every row held - // without one): by arrival rate, at most the rows they hold. - let retained_raw_rows = |lookback_ms: Option| -> Option { + // The raw rows of its sources back `lookback_ms` (every row held + // without one), which a scan reads or keeps: by arrival rate, at most + // the rows they hold. + let raw_rows_back = |lookback_ms: Option| -> Option { match raw_rows(lookback_ms) { (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), (by_rate, rows_held) => by_rate.or(rows_held), @@ -514,13 +515,11 @@ pub fn node_resource_usage_and_cost<'a>( // oldest row read, its `retention_ms` ($h_v$; an offset // window keeps more than it reads), at most those held. if !ingestion || raw_rows_materialized_at_ingestion { - raw_retained_rows = retained_raw_rows(node.retention_ms); + raw_retained_rows = raw_rows_back(node.retention_ms); } - let (by_rate, rows_held) = raw_rows(lookback); - let rows = match (ingestion, by_rate, rows_held) { - (true, by_rate, _) => by_rate.ok_or_else(|| missing_rate(source))?, - (false, Some(by_rate), Some(rows_held)) => by_rate.min(rows_held), - (false, by_rate, rows_held) => by_rate.or(rows_held).ok_or_else(|| { + let rows = match ingestion { + true => raw_rows(lookback).0.ok_or_else(|| missing_rate(source))?, + false => raw_rows_back(lookback).ok_or_else(|| { AnalyticalCostError::MissingSourceFact { source_name: source.to_string(), missing: match lookback { @@ -650,7 +649,7 @@ pub fn node_resource_usage_and_cost<'a>( // second of them. let out = match raw_rows_materialized_at_ingestion { true => { - raw_retained_rows = retained_raw_rows(node.retention_ms); + raw_retained_rows = raw_rows_back(node.retention_ms); let source = sources.first().copied().unwrap_or("(none)"); edge(raw_retained_rows.ok_or(retained_without_a_bound(source))?)? } diff --git a/crates/cost/src/plan/execution_demand/mod.rs b/crates/cost/src/plan/execution_demand/mod.rs index 9c336edb6..eaeb77a6e 100644 --- a/crates/cost/src/plan/execution_demand/mod.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -15,7 +15,6 @@ use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; use asap_types::stages::Lifetime; use asap_types::workload::{ greatest_common_divisor, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand, - TimestampMs, }; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -299,61 +298,45 @@ pub enum Consumer { Materialized(PhysicalASAPNodeId), } -/// The consumers of each node of `dag` not materialized across evaluations -/// that any consumer reaches ([`computations`]): the queries and the nodes -/// materialized across evaluations whose walk down their inputs reaches it -/// before meeting such a node, each once; the queries first, in order, -/// then the materialized nodes in `dag`'s node order. A `OneEvaluation` -/// node is computed within its consumers' context, so a walk goes through -/// it. -pub fn consumers( - dag: &PhysicalASAPDAG, +/// How each query reads the nodes below it, in a plan or a memo: per node, +/// each (query, anchor) whose walk from the query's `starts` down +/// `inputs_of` reaches it, once, skipping (and stopping at) a node +/// `stops_at` keeps. Below a `TimeShift` anchored with `@ end()` or +/// `@ start()` (`anchor_of`) a query reads per request, with that anchor, +/// unless the anchor is its evaluation time ([`anchor_is_evaluation_time`] +/// over the queries on its grid that reach the shift through any node); +/// elsewhere with none. Stage 3 walks memo groups with it and +/// [`consumers`] plan nodes, so both rate a context by the queries whose +/// walk reaches it. Stage 3 walks every candidate of a group, so a query +/// reading a group through one candidate counts for the group's other +/// candidates too. +/// +/// Example: an instant `sum_over_time(m[5m] @ end())` and a range +/// `sum_over_time(m[5m] @ end() offset 10m)`, both every 60 s: `m[5m]` is +/// read by the instant query with no anchor and by the range query with +/// `@ end()` (its requests' ends are not its evaluations). +pub fn query_reads( + starts: &[Vec], + inputs_of: impl Fn(N) -> Vec, + anchor_of: impl Fn(N) -> Option, + stops_at: impl Fn(N) -> bool, entries: &[QueryWorkloadEntry], -) -> HashMap> { - let materialized_nodes: Vec = dag - .nodes - .iter() - .filter(|n| matches!(n.lifetime, Lifetime::AcrossEvaluations(_))) - .map(|n| n.id) - .collect(); - let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); - let inputs = |id: PhysicalASAPNodeId| { - dag.edges - .iter() - .filter(move |e| e.consumer == id) - .map(|e| e.producer) - }; - // The anchor of each `TimeShift` anchored with `@ end()` or `@ start()`. - let request_anchor: HashMap = dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { - Some(at @ (AtModifier::End | AtModifier::Start)) => Some((node.id, at)), - _ => None, - }, - _ => None, - }) - .collect(); - let starts = query_starts(dag); +) -> HashMap)>> { // The queries reaching each anchored shift, through any node. - let mut reaching_shift: HashMap> = HashMap::new(); + let mut reaching_shift: HashMap> = HashMap::new(); for (query, start) in starts.iter().enumerate() { let mut seen = HashSet::new(); let mut stack = start.clone(); while let Some(id) = stack.pop() { if seen.insert(id) { - if request_anchor.contains_key(&id) { + if anchor_of(id).is_some() { reaching_shift.entry(id).or_default().insert(query); } - stack.extend(inputs(id)); + stack.extend(inputs_of(id)); } } } - // Whether query `query`'s input of shift `shift` anchored with `at` - // runs in its evaluation context ([`anchor_is_evaluation_time`] over - // the queries on its grid reaching the shift). - let in_evaluation_context = |shift: PhysicalASAPNodeId, query: usize, at: AtModifier| { + let in_evaluation_context = |shift: N, query: usize, at: AtModifier| { let Some(entry) = entries.get(query) else { return false; }; @@ -364,37 +347,107 @@ pub fn consumers( ); anchor_is_evaluation_time(at, on_grid.iter().map(|&q| &entries[q])) }; - let walks = starts - .into_iter() - .enumerate() - .map(|(query, start)| (Consumer::Query(query), start)) - .chain( - materialized_nodes - .iter() - .map(|&id| (Consumer::Materialized(id), inputs(id).collect())), - ); - let mut consumers: HashMap<_, Vec> = HashMap::new(); - for (consumer, start) in walks { + let mut reads: HashMap)>> = HashMap::new(); + for (query, start) in starts.iter().enumerate() { let mut seen = HashSet::new(); - let mut stack: Vec<(PhysicalASAPNodeId, Consumer)> = - start.into_iter().map(|id| (id, consumer)).collect(); - while let Some((id, consumer)) = stack.pop() { - if is_materialized.contains(&id) || !seen.insert((id, consumer)) { + let mut stack: Vec<(N, Option)> = start.iter().map(|&id| (id, None)).collect(); + while let Some((id, anchor)) = stack.pop() { + if stops_at(id) || !seen.insert((id, anchor)) { continue; } - consumers.entry(id).or_default().push(consumer); - // A query's input of an `@ end()` / `@ start()` shift runs once - // per request, or in its evaluation context when the anchor is - // its evaluation time. - let below = match (consumer, request_anchor.get(&id)) { - (Consumer::Query(query), Some(&at)) if !in_evaluation_context(id, query, at) => { - Consumer::QueryRequest { query, at } - } - (consumer, _) => consumer, + reads.entry(id).or_default().push((query, anchor)); + let below = match (anchor, anchor_of(id)) { + (None, Some(at)) if !in_evaluation_context(id, query, at) => Some(at), + (anchor, _) => anchor, }; - stack.extend(inputs(id).map(|input| (input, below))); + stack.extend(inputs_of(id).into_iter().map(|input| (input, below))); + } + } + reads +} + +/// The anchor of `operator` when it is a `TimeShift` anchored with +/// `@ end()` or `@ start()` ([`query_reads`]). +pub fn request_anchor(operator: &Operator) -> Option { + match operator { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { + Some(at @ (AtModifier::End | AtModifier::Start)) => Some(at), + _ => None, + }, + _ => None, + } +} + +/// The consumers of each node of `dag` not materialized across evaluations +/// that any consumer reaches ([`computations`]): the queries +/// ([`query_reads`]) and the nodes materialized across evaluations whose +/// walk down their inputs reaches it before meeting such a node, each +/// once, sorted. A `OneEvaluation` node is computed within its consumers' +/// context, so a walk goes through it. +pub fn consumers( + dag: &PhysicalASAPDAG, + entries: &[QueryWorkloadEntry], +) -> HashMap> { + let materialized_nodes: Vec = dag + .nodes + .iter() + .filter(|n| matches!(n.lifetime, Lifetime::AcrossEvaluations(_))) + .map(|n| n.id) + .collect(); + let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); + let inputs = |id: PhysicalASAPNodeId| -> Vec { + dag.edges + .iter() + .filter(|e| e.consumer == id) + .map(|e| e.producer) + .collect() + }; + let payload: HashMap> = dag + .nodes + .iter() + .map(|node| (node.id, &node.payload)) + .collect(); + let mut consumers: HashMap<_, Vec> = HashMap::new(); + let queries = query_reads( + &query_starts(dag), + inputs, + |id| { + payload + .get(&id) + .and_then(|operator| request_anchor(operator)) + }, + |id| is_materialized.contains(&id), + entries, + ); + for (id, reads) in queries { + consumers + .entry(id) + .or_default() + .extend(reads.into_iter().map(|read| match read { + (query, None) => Consumer::Query(query), + (query, Some(at)) => Consumer::QueryRequest { query, at }, + })); + } + // Each node materialized across evaluations, over the nodes it reads + // down to the next such node. + for &materialized in &materialized_nodes { + let mut seen = HashSet::new(); + let mut stack = inputs(materialized); + while let Some(id) = stack.pop() { + if is_materialized.contains(&id) || !seen.insert(id) { + continue; + } + consumers + .entry(id) + .or_default() + .push(Consumer::Materialized(materialized)); + stack.extend(inputs(id)); } } + // Queries first, in order, then the materialized nodes. + for of_node in consumers.values_mut() { + of_node.sort(); + } consumers } @@ -406,6 +459,9 @@ pub fn consumers( /// every evaluation of the grid serves the range queries' request ends /// too). Otherwise it runs once per request, in a context of its own. /// +/// `@ start()` of an instant query is its evaluation time too; it keeps +/// its request context here, as the user decided for `@ end()` only. +/// /// Example: `sum_over_time(m[5m] @ end())` and `sum_over_time(m[5m])`, /// both instant every 60 s: true, so `m[5m]` is computed once per minute /// for both; the same `@ end()` in a range query alone: false. @@ -469,10 +525,11 @@ pub fn request_time_grid_ms(entry: &QueryWorkloadEntry, at: AtModifier) -> u64 { /// so): with no `anchor`, at every evaluation of theirs, a range query's /// steps each one ([`evaluation_rate`]); below a `TimeShift` anchored with /// `anchor`, once per request: below `@ end()` once per distinct request -/// end; below `@ start()` once per distinct (request schedule, span), as -/// two queries requested together with different spans need the input at -/// two start times, and a result not kept across evaluations serves only -/// the request it is computed for. +/// end ([`evaluation_rate`] of their request schedules); below +/// `@ start()` that per span, summed over the distinct spans, as two +/// queries requested together with different spans need the input at two +/// start times, and a result not kept across evaluations serves only the +/// request it is computed for. /// /// Example: two range queries every 60 s at phase 0 with steps of 30 s, /// spans 1 h and 2 m: below `@ start()` 2/60 per second (one run per @@ -483,65 +540,37 @@ pub fn context_executions_per_second<'e>( queries: impl IntoIterator, non_repeating_query_horizon_seconds: f64, ) -> Result { - let queries: Vec<&QueryWorkloadEntry> = queries.into_iter().collect(); - if anchor == Some(AtModifier::Start) { - // One run per request of each distinct (schedule, span). - let mut runs: Vec<(&QueryRecurrence, Option)> = Vec::new(); - let mut rate = 0.0; - for entry in queries { - let key = ( - &entry.recurrence, - entry.range_evaluation.map(|range| range.span_ms), - ); - if runs.contains(&key) { - continue; + let queries = queries.into_iter(); + match anchor { + // Requests of one span need the input at one start time each; + // different spans need it at different times of one request. + Some(AtModifier::Start) => { + let mut by_span: BTreeMap, Vec<&QueryRecurrence>> = BTreeMap::new(); + for entry in queries { + by_span + .entry(entry.range_evaluation.map(|range| range.span_ms)) + .or_default() + .push(&entry.recurrence); } - runs.push(key); - rate += evaluation_rate( - [(&anchored_request_recurrence(entry, AtModifier::Start), None)], - non_repeating_query_horizon_seconds, - )?; + by_span + .values() + .map(|recurrences| { + evaluation_rate( + recurrences.iter().map(|&recurrence| (recurrence, None)), + non_repeating_query_horizon_seconds, + ) + }) + .sum() } - return Ok(rate); - } - let per_evaluation: Vec<(QueryRecurrence, Option)> = queries - .into_iter() - .map(|entry| match anchor { - None => (entry.recurrence.clone(), entry.range_evaluation), - Some(at) => (anchored_request_recurrence(entry, at), None), - }) - .collect(); - evaluation_rate( - per_evaluation - .iter() - .map(|(recurrence, range)| (recurrence, *range)), - non_repeating_query_horizon_seconds, - ) -} - -/// The recurrence of the times `entry`'s requests anchor `at` at: its own, -/// with a fixed interval's phase moved back by the range-evaluation span -/// for `@ start()`. -fn anchored_request_recurrence(entry: &QueryWorkloadEntry, at: AtModifier) -> QueryRecurrence { - match (&entry.recurrence, at, entry.range_evaluation) { - ( - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }), - AtModifier::Start, - Some(range), - ) if interval.0 > 0 => { - let interval_ms = u64::from(interval.0); - let phase_ms = (evaluation_phase.0 % interval_ms + interval_ms - - range.span_ms % interval_ms) - % interval_ms; - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: *interval, - evaluation_phase: TimestampMs(phase_ms), - }) - } - (recurrence, _, _) => recurrence.clone(), + // Once per request end, however many steps the request evaluates. + Some(_) => evaluation_rate( + queries.map(|entry| (&entry.recurrence, None)), + non_repeating_query_horizon_seconds, + ), + None => evaluation_rate( + queries.map(|entry| (&entry.recurrence, entry.range_evaluation)), + non_repeating_query_horizon_seconds, + ), } } diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs index af83a6a66..c2fe3f42a 100644 --- a/crates/cost/src/plan/execution_demand/tests.rs +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -356,9 +356,10 @@ fn a_shared_one_evaluation_at_end_shift_runs_its_input_once_per_request() { /// Two range queries every 60 s at a 30 s step, over 1 h and over 90 s, /// below one `@ start()` shift: one context (one evaluation grid, one /// anchor), whose requests start at `k · 60 s` and `k · 60 s + 30 s`: the -/// input runs 2/60 per second. Over 1 h and over 2 m, both start at -/// `k · 60 s` but at different times of one request: still 2/60. The same -/// query twice runs once. +/// input runs 2/60 per second. Over 1 h and over 2 m, one request needs +/// the input at two start times, its end less 1 h and its end less 2 m: +/// still 2/60. The same query twice runs once, and so does one schedule +/// written at phases 10 s and 70 s. #[test] fn at_start_inputs_run_once_per_request_of_each_span() { let with_span = |span_ms| QueryWorkloadEntry { @@ -396,6 +397,15 @@ fn at_start_inputs_run_once_per_request_of_each_span() { context_executions_per_second(Some(AtModifier::Start), &twice, 3600.0).unwrap(), 1.0 / 60.0, ); + let mut phase_70s = with_span(120_000); + phase_70s.recurrence = fixed(60_000, 70_000); + let mut phase_10s = with_span(120_000); + phase_10s.recurrence = fixed(60_000, 10_000); + close( + context_executions_per_second(Some(AtModifier::Start), &[phase_10s, phase_70s], 3600.0) + .unwrap(), + 1.0 / 60.0, + ); } // Invalid fixed intervals and estimated rates still produce the evaluator's error. diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index 24809a696..34934bc6e 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -539,3 +539,41 @@ fn the_objective_is_the_plan_cost_on_two_phases_of_one_grid() { objective_and_plan_cost(&w, &CostInputs::default(), true), ); } + +/// An instant `sum_over_time(m[5m] @ end())` and a range +/// `sum_over_time(m[5m] @ end() offset 10m)` over the last 10 minutes, +/// both every 60 s: `m[5m]` is read by the instant query in its +/// evaluation context and by the range query once per request below its +/// shift, so a run in the evaluation context is rated by the instant query +/// only, in the MILP and in `plan_cost` alike (the objective was 0.3642/s +/// for a plan `plan_cost` charged 0.1542/s). +#[test] +fn the_objective_is_the_plan_cost_on_an_instant_and_a_shifted_range_at_end() { + let w: PlanningWorkload = serde_json::from_value(serde_json::json!({ + "query_workload": {"language": "promql", "repeating_queries": [ + requested("sum_over_time(m[5m] @ end())", 60_000, None), + requested( + "sum_over_time(m[5m] @ end() offset 600s)", + 60_000, + Some((60_000, 600_000)), + ), + ]}, + "data_workload": { + "arrival": {"ingesting": {"rows_per_second": 2000.0}}, + "sources": { + "m": {"series_count": 1000, "metric_type": "gauge", "sample_interval": 1000}, + "c": {"series_count": 1000, "metric_type": "counter", "sample_interval": 1000}, + }, + }, + })) + .unwrap(); + let capabilities = DeploymentCapabilities { + supported_summaries: Some(asap_executor::supported_summary_families()), + ..Default::default() + }; + let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); + assert_equal( + "instant and shifted range @ end()", + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1, + ); +} diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 91e0e9c91..f6da05729 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -72,7 +72,10 @@ use asap_types::stages::{ use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::plan::candidate_node; -use crate::retention::{edge_retention_need_ms, edge_scan_read_ms, ms_to_seconds, reads_back}; +use crate::retention::{ + edge_retention_need_ms, edge_scan_read_ms, least_need_ms_per_execution, ms_to_seconds, + reads_back, +}; use crate::selectable::{Execution, SelectableExecutions}; /// The objective's coefficients @@ -98,6 +101,10 @@ pub(crate) struct ExecutionCosts { /// `None` when its producer retains no rows or the edge reads it /// unbounded. pub(crate) retention_need_ms_per_edge: Vec>, + /// The least need, in ms, over each execution's costed reads, by + /// position in `SelectableExecutions::executions` + /// ([`least_need_ms_per_execution`]). + pub(crate) least_need_ms_per_execution: Vec>, /// The source of each query-time `Scan`: its retention is raw data on /// disk, which a plan keeps once per source for the selected scan that /// keeps the most, as `plan_cost` charges it ($r_s$). @@ -326,7 +333,11 @@ pub(crate) fn execution_costs( let own_execution: HashMap = executions .iter() .enumerate() - .filter(|(_, execution)| is_across_evaluations(physical, execution.candidate)) + .filter(|(_, execution)| { + physical.physical_candidates[execution.candidate.0] + .physical_properties + .is_across_evaluations() + }) .map(|(index, execution)| (execution.candidate, index)) .collect(); // π_v of a candidate retaining rows every consumer reads within a bound: @@ -351,7 +362,10 @@ pub(crate) fn execution_costs( .max(1); let longer = node_retaining(v, Some(retention_ms + delta_ms)); let delta_seconds = ms_to_seconds(delta_ms); - let per_retained_second = match is_across_evaluations(physical, v) { + let per_retained_second = match physical.physical_candidates[v.0] + .physical_properties + .is_across_evaluations() + { true => own_execution.get(&v).and_then(|&index| { let (own, raw_data, per_second) = at_rate[index].as_ref().ok()?; let inputs = baseline[v.0].as_ref()?; @@ -406,9 +420,12 @@ pub(crate) fn execution_costs( Ok(if query_time_scan[v.0] { // A query-time scan's raw data is charged per source. own - } else if let (Some(per_retained_second), true) = - (per_retained_second, is_across_evaluations(physical, v)) - { + } else if let (Some(per_retained_second), true) = ( + per_retained_second, + physical.physical_candidates[v.0] + .physical_properties + .is_across_evaluations(), + ) { let retention_seconds = ms_to_seconds(most_read_ms[&v].unwrap_or(0)); own + raw_data - per_retained_second * retention_seconds } else { @@ -416,7 +433,7 @@ pub(crate) fn execution_costs( }) }) .collect(); - let per_edge = selectable + let per_edge: Vec> = selectable .edges() .iter() .zip(&scan_read_ms_per_edge) @@ -453,23 +470,19 @@ pub(crate) fn execution_costs( Some(with_u_own + with_u_raw_data - own - raw_data + scan_read) }) .collect(); + let least_need_ms_per_execution = + least_need_ms_per_execution(selectable, &per_edge, &retention_need_ms_per_edge, entries); ExecutionCosts { per_execution, per_edge, retention_cost_per_retained_second, retention_need_ms_per_edge, + least_need_ms_per_execution, raw_data_source, unbounded_raw_data_on_disk_cost_per_second, } } -/// Whether candidate `v` is materialized across evaluations. -fn is_across_evaluations(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { - physical.physical_candidates[v.0] - .physical_properties - .is_across_evaluations() -} - /// Whether candidate `v` is a `Scan` at query time, which reads raw data /// in each execution. fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { @@ -487,12 +500,15 @@ fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidat } /// $\rho$ of `execution`'s context, how often it runs, from the queries it -/// serves: in an evaluation context, as often as that context runs for -/// the queries on its grid that reach the execution's group -/// ([`context_executions_per_second`], which `plan_cost` uses too): at -/// their evaluations, or, for the inputs of a `TimeShift` anchored with -/// `@`, once per request of the queries on the shift's grid that reach the -/// shift; +/// serves ([`context_executions_per_second`], which `plan_cost` uses too): +/// in an evaluation context, for the queries on its grid whose walk over +/// the memo reaches the execution's group in their evaluation context +/// (`SelectableExecutions::readers_of`, [`asap_cost::plan::execution_demand::query_reads`], +/// the walk `plan_cost` takes over the plan), at their evaluations; for +/// the inputs of a `TimeShift` anchored with `@`, once per request of the +/// queries on the shift's grid that reach the shift. The memo walk goes +/// through every candidate of a group, so a query reading a group through +/// one candidate counts for its other candidates too; /// in `Materialized(m)`, as often as $m$ runs /// ([`asap_cost::plan::execution_demand::executions_per_second`]); in the /// ingestion pipeline, once per second of arriving rows. @@ -507,9 +523,10 @@ pub(crate) fn executions_per_second( non_repeating_query_horizon_seconds: f64, ) -> Result { let memo = &physical.logical_candidates.memo; + let group_of = + |v: PhysicalCandidateId| physical.physical_candidates[v.0].logical_candidate_id.group; let reaching = |v: PhysicalCandidateId| { - let group = physical.physical_candidates[v.0].logical_candidate_id.group; - memo.groups()[group.0 as usize] + memo.groups()[group_of(v).0 as usize] .reached_by_queries() .iter() .copied() @@ -532,7 +549,7 @@ pub(crate) fn executions_per_second( None, entries_of(queries_of_context( grid.period_ms, - reaching(execution.candidate), + selectable.readers_of(group_of(execution.candidate), |anchor| anchor.is_none()), entries, )), non_repeating_query_horizon_seconds, diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index 7caa0e268..8f893b93b 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -4,9 +4,9 @@ //! per constraint family ([`model`]), the retention amount $h_v$ and //! constraint (R) ([`retention`]), each query's exact-only reads, //! constraint (X) ([`exact_only`]), the executions other executions -//! dominate ([`dominance`]), the graph algorithm it needs -//! ([`graph`]), the bound it scales by ([`upper_bound`]), and the accuracy -//! extension ([`accuracy`]). +//! dominate ([`dominance`]), the graph algorithm it needs ([`graph`]), +//! the bound that leaves out executions dearer than a feasible plan before +//! scaling ([`upper_bound`]), and the accuracy extension ([`accuracy`]). mod accuracy; mod dominance; pub(crate) mod exact_only; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 221d30603..93586f602 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -12,8 +12,9 @@ //! [`solve_selection`] solves it without (K), whose variables and //! constraint are in [`super::accuracy`], `todo!()` until it is added. The //! strongly connected components (E) needs are in [`super::graph`]; the -//! bound that leaves out executions before scaling in -//! [`super::upper_bound`]. +//! executions left out before scaling, those dearer than a feasible plan +//! and those their `OneEvaluation` twin dominates, in +//! [`super::upper_bound`] and [`super::dominance`]. use std::collections::{BTreeMap, HashMap}; use std::time::Instant; @@ -200,7 +201,7 @@ fn query_coverage_constraints( /// $\sum y + \sum_q z_{q,v} = x_{\kappa,v}$ ([`use_constraints`]), which /// with $y, z \ge 0$ implies (B) and (D) for it, so those rows are left /// out. -fn has_one_reader(inputs: &SelectionInputs<'_>, index: usize) -> bool { +pub(super) fn has_one_reader(inputs: &SelectionInputs<'_>, index: usize) -> bool { let v = inputs.selectable.executions()[index].candidate; has_one_reader_per_context(inputs.physical.physical_candidates[v.0].physical_properties) } @@ -309,8 +310,8 @@ fn uses_of(inputs: &SelectionInputs<'_>, variables: &SelectionVariables) -> Vec< pub(crate) struct SolvedSelection { /// The selected executions, ascending. pub(crate) executions: Vec, - /// The selected edges as (consumer, edge, producer), ascending. - pub(crate) edges: Vec<(Execution, PhysicalEdgeId, Execution)>, + /// The selected edges, ascending. + pub(crate) edges: Vec, /// The execution answering each of `SelectableExecutions::answering`, /// in its order. pub(crate) outputs: Vec, @@ -327,6 +328,15 @@ pub(crate) struct SolvedSelection { pub(crate) solve_ms: u64, } +/// A selected $y_{\kappa,u,c,i}$: edge `edge` from execution `producer` +/// into execution `consumer`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SelectedEdge { + pub(crate) consumer: Execution, + pub(crate) edge: PhysicalEdgeId, + pub(crate) producer: Execution, +} + /// Why the MILP has no optimum. #[derive(Debug, Clone, PartialEq)] pub(crate) enum SelectionMilpError { @@ -503,7 +513,11 @@ pub(crate) fn solve_selection( .map(|&position| { let edge = selectable.edges()[position]; let (consumer, producer) = (edge.consumer, edge.producer); - (executions[consumer], edge.edge, executions[producer]) + SelectedEdge { + consumer: executions[consumer], + edge: edge.edge, + producer: executions[producer], + } }) .collect(), outputs: answering diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 46c9a502b..c67b900a9 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -30,7 +30,7 @@ use super::*; use crate::cost::ExecutionCosts; use crate::milp::exact_only::must_not_read; use crate::plan::plan_of; -use crate::retention::edge_retention_need_ms; +use crate::retention::{edge_retention_need_ms, least_need_ms_per_execution}; use crate::selectable::{output_groups, SelectableEdge}; const NONE: Lifetime = Lifetime::None; @@ -156,7 +156,7 @@ fn selection_inputs( .map(|&(v, per_retained_second)| (PhysicalCandidateId(v), per_retained_second)) .collect(); let edge_of = |edge: &SelectableEdge| physical.edge(edge.edge); - let costs = ExecutionCosts { + let mut costs = ExecutionCosts { per_execution: selectable .executions() .iter() @@ -188,9 +188,16 @@ fn selection_inputs( }) .collect(), retention_cost_per_retained_second, + least_need_ms_per_execution: Vec::new(), raw_data_source: HashMap::new(), unbounded_raw_data_on_disk_cost_per_second: HashMap::new(), }; + costs.least_need_ms_per_execution = least_need_ms_per_execution( + &selectable, + &costs.per_edge, + &costs.retention_need_ms_per_edge, + entries, + ); (selectable, costs) } @@ -255,7 +262,7 @@ fn selects_the_producer_whose_own_and_edge_costs_are_lowest() { let solved = solved.unwrap(); assert_eq!(candidates(&solved), vec![1, 2]); assert_eq!(solved.edges.len(), 1); - assert_eq!(solved.edges[0].1, edge(&physical, 1, 2)); + assert_eq!(solved.edges[0].edge, edge(&physical, 1, 2)); assert_eq!(solved.outputs[0].candidate, PhysicalCandidateId(2)); let plan = plan_of(&physical, &solved, &output_groups(&physical)).unwrap(); diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs index 92820c218..faca6e1ab 100644 --- a/crates/stage3-plan-selection/src/milp/retention.rs +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -52,9 +52,10 @@ pub(crate) fn retention_variables( /// read when its evaluation starts. /// /// With them, per execution $u$ of such a $v$, $h_v \ge -/// \min_{\text{uses}} \operatorname{need} \cdot x_u$ over the reads that -/// may select it: (F) gives a selected execution a use, and (R) then -/// retains at least the least need. It is implied for integer $x$, $y$, +/// \min_{\text{uses}} \operatorname{need} \cdot x_u$ over its costed +/// reads (`ExecutionCosts::least_need_ms_per_execution`): (F) gives a +/// selected execution a use, and (R) then retains at least the least +/// need. It is implied for integer $x$, $y$, /// $z$, but tightens the relaxation where fractional $y$ leave $h_v$ low. pub(crate) fn retention_constraints( inputs: &SelectionInputs<'_>, @@ -65,8 +66,6 @@ pub(crate) fn retention_constraints( ) -> Vec { let executions = inputs.selectable.executions(); let mut constraints = Vec::new(); - // The least need over each execution's uses, for one with $h_v$. - let mut least_need_ms: BTreeMap = BTreeMap::new(); for (index, reader, need_ms) in reads_back( inputs.selectable, &inputs.costs.retention_need_ms_per_edge, @@ -82,14 +81,13 @@ pub(crate) fn retention_constraints( if let Some(&read) = read { let need_ms = need_ms.expect("a candidate with h_v is read within a bound"); constraints.push(constraint!(h >= ms_to_seconds(need_ms) * read)); - let least = least_need_ms.entry(index).or_insert(need_ms); - *least = (*least).min(need_ms); } } - for (index, need_ms) in least_need_ms { - if need_ms > 0 { - let h = h[&executions[index].candidate]; - constraints.push(constraint!(h >= ms_to_seconds(need_ms) * x[index])); + for (index, need_ms) in inputs.costs.least_need_ms_per_execution.iter().enumerate() { + if let (Some(&h), Some(need_ms)) = (h.get(&executions[index].candidate), *need_ms) { + if need_ms > 0 { + constraints.push(constraint!(h >= ms_to_seconds(need_ms) * x[index])); + } } } constraints diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index c363f9aa9..6536fbf5c 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -15,9 +15,8 @@ use std::collections::{BTreeMap, HashMap}; use super::exact_only::must_not_read; -use super::model::{selection_cost_per_second, SelectionInputs}; -use crate::retention::{reads_back, Reader}; -use crate::selectable::{has_one_reader_per_context, QueryId}; +use super::model::{has_one_reader, selection_cost_per_second, SelectionInputs}; +use crate::selectable::QueryId; /// How many times the greedy plan is rebuilt without an execution that /// made it infeasible before giving up on a bound. @@ -43,28 +42,11 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec> = vec![None; count]; - for (index, reader, need_ms) in reads_back( - selectable, - &costs.retention_need_ms_per_edge, - inputs.entries, - ) { - let costed = match reader { - Reader::Edge(position) => costs.per_edge[position].is_some(), - Reader::Output(_) => true, - }; - if let (true, Some(need_ms)) = (costed, need_ms) { - let least = &mut least_need_ms[index]; - *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); - } - } + // The least retention (R) makes each execution pay; a query-time + // scan's retention is its source's $r_s$, left out. let least_retention_cost = |index: usize| { let v = executions[index].candidate; - match least_need_ms[index] { + match costs.least_need_ms_per_execution[index] { Some(need_ms) if !costs.raw_data_source.contains_key(&v) => { costs.retention_cost_per_second(v, need_ms) } @@ -109,13 +91,8 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { let selectable = inputs.selectable; let executions = selectable.executions(); let count = executions.len(); - let one_reader: Vec = executions - .iter() - .map(|execution| { - has_one_reader_per_context( - inputs.physical.physical_candidates[execution.candidate.0].physical_properties, - ) - }) + let one_reader: Vec = (0..count) + .map(|index| has_one_reader(inputs, index)) .collect(); let forbidden = |query: QueryId, e: usize| must_not_read(inputs.physical, query, executions[e].candidate); diff --git a/crates/stage3-plan-selection/src/plan.rs b/crates/stage3-plan-selection/src/plan.rs index e25fc8f0f..466ede909 100644 --- a/crates/stage3-plan-selection/src/plan.rs +++ b/crates/stage3-plan-selection/src/plan.rs @@ -16,7 +16,7 @@ use asap_types::plan::physical_asap_dag::{ use asap_types::plan::view_coverage; use asap_types::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId}; -use crate::milp::model::SolvedSelection; +use crate::milp::model::{SelectedEdge, SolvedSelection}; use crate::selectable::{Execution, QueryId}; /// Candidate `v` as a plan node with id `id`, its input `i` read from node @@ -71,7 +71,12 @@ pub(crate) fn plan_of( ) -> Result { let mut producer_of: HashMap<(Execution, usize), Execution> = HashMap::new(); let mut consumers_of: BTreeMap> = BTreeMap::new(); - for &(consumer, edge, producer) in &solved.edges { + for &SelectedEdge { + consumer, + edge, + producer, + } in &solved.edges + { producer_of.insert( (consumer, physical.edge(edge).input_index_of_consumer), producer, diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 40f7582e3..d965d80dd 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -11,7 +11,7 @@ use asap_types::ir::{NonASAPOp, Operator}; use asap_types::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; use asap_types::stages::{ - MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, + MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, }; use asap_types::workload::QueryWorkloadEntry; @@ -99,6 +99,32 @@ pub(crate) fn reads_back<'a>( edges.chain(outputs) } +/// The least need, in ms, over each execution's costed reads +/// ([`reads_back`]: an edge out of it `per_edge` costs, or a query output +/// it may answer), by position in [`SelectableExecutions::executions`]; +/// `None` without a bounded one. (F) gives a selected execution a use and +/// (R) then retains at least that: the greedy bound charges it, and the +/// MILP has $h_v \ge$ it $\cdot x$. +pub(crate) fn least_need_ms_per_execution( + selectable: &SelectableExecutions, + per_edge: &[Option], + need_ms_per_edge: &[Option], + entries: &[QueryWorkloadEntry], +) -> Vec> { + let mut least_need_ms = vec![None; selectable.executions().len()]; + for (index, reader, need_ms) in reads_back(selectable, need_ms_per_edge, entries) { + let costed = match reader { + Reader::Edge(position) => per_edge[position].is_some(), + Reader::Output(_) => true, + }; + if let (true, Some(need_ms)) = (costed, need_ms) { + let least: &mut Option = &mut least_need_ms[index]; + *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); + } + } + least_need_ms +} + /// `ms` in seconds, the unit of $h_v$ in the MILP and of the costs per /// second retained. pub(crate) fn ms_to_seconds(ms: u64) -> f64 { @@ -114,31 +140,22 @@ fn read_back( edge: PhysicalEdgeId, span_ms: impl Fn(PhysicalCandidateId) -> u64, ) -> Option { - let edge = physical.edge(edge); - let producer = edge.producer_candidate_id; - let readers_of = |v: PhysicalCandidateId| match v == producer { - true => vec![edge.consumer_candidate_id], - false => physical - .admissible_edges_out_of(v) - .iter() - .map(|&out| physical.edge(out).consumer_candidate_id) - .collect(), - }; let candidates = Candidates { physical, - readers_of: &readers_of, + edge: physical.edge(edge), }; - asap_cost::read_back_ms(&candidates, producer, span_ms) + asap_cost::read_back_ms(&candidates, candidates.edge.producer_candidate_id, span_ms) } -/// Stage 3's candidates as the time walks see them: the readers a caller -/// gives (those that may read a candidate, or the selected ones), each -/// candidate's admissible producers as its inputs, its operator at its -/// timing and lifetime ([`InputTimeAccess::of`]), and a current-time read -/// of a `Scan` (raw data) needing every row. +/// Stage 3's candidates as the time walks see them above `edge`: its +/// producer read by its consumer only, any other candidate by every +/// candidate that may read it (its admissible edges out); each candidate's +/// admissible producers as its inputs, its operator at its timing and +/// lifetime ([`InputTimeAccess::of`]), and a current-time read of a +/// `Scan` (raw data) needing every row. struct Candidates<'a> { physical: &'a PhysicalASAPMemoCandidates, - readers_of: &'a dyn Fn(PhysicalCandidateId) -> Vec, + edge: &'a PhysicalEdge, } impl TimeIndexedPlan for Candidates<'_> { @@ -146,7 +163,15 @@ impl TimeIndexedPlan for Candidates<'_> { type Child = MemoGroupId; fn readers_of(&self, v: PhysicalCandidateId) -> Vec { - (self.readers_of)(v) + match v == self.edge.producer_candidate_id { + true => vec![self.edge.consumer_candidate_id], + false => self + .physical + .admissible_edges_out_of(v) + .iter() + .map(|&out| self.physical.edge(out).consumer_candidate_id) + .collect(), + } } fn inputs_of(&self, v: PhysicalCandidateId) -> Vec { diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index 99b1d9574..29a24844f 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -32,7 +32,8 @@ use std::collections::{BTreeSet, HashMap}; use asap_cost::plan::execution_demand::{ - anchor_is_evaluation_time, evaluation_grid_ms, queries_of_context, request_time_grid_ms, + anchor_is_evaluation_time, evaluation_grid_ms, queries_of_context, query_reads, request_anchor, + request_time_grid_ms, }; use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; @@ -98,6 +99,10 @@ pub(crate) struct SelectableExecutions { request_anchor_of_context: Vec, /// The id of each context with its request anchor. context_ids: HashMap<(ExecutionContext, RequestAnchor), ExecutionContextId>, + /// Per memo group, each (query, anchor) whose walk over the memo reaches + /// it ([`query_reads`]): the anchor of the `TimeShift` above it it is + /// read per request below, `None` in the query's evaluation context. + reads_of_group: HashMap)>>, /// Every execution reached from a query output, ascending. executions: Vec, /// Every $y_{\kappa,u,c,i}$, ascending. @@ -129,6 +134,7 @@ impl SelectableExecutions { outputs: &[(QueryId, MemoGroupId)], ) -> Self { let mut selectable = Self { + reads_of_group: reads_of_groups(physical, entries, outputs), contexts: Vec::new(), request_anchor_of_context: Vec::new(), context_ids: HashMap::new(), @@ -306,17 +312,11 @@ impl SelectableExecutions { else { return consumer.context; }; - let queries: Vec = queries_of_context( - grid.period_ms, - memo.groups()[group.0 as usize] - .reached_by_queries() - .iter() - .copied(), - entries, - ) - .into_iter() - .map(QueryId) - .collect(); + let queries: Vec = + queries_of_context(grid.period_ms, self.readers_of(group, |_| true), entries) + .into_iter() + .map(QueryId) + .collect(); if anchor_is_evaluation_time(at, queries.iter().map(|query| &entries[query.0])) { return consumer.context; } @@ -337,6 +337,22 @@ impl SelectableExecutions { } } + /// The queries whose walk reaches `group` ([`Self::reads_of_group`]), + /// each once, reading it with an anchor `anchored` keeps. + pub(crate) fn readers_of( + &self, + group: MemoGroupId, + anchored: impl Fn(Option) -> bool, + ) -> BTreeSet { + self.reads_of_group + .get(&group) + .into_iter() + .flatten() + .filter(|(_, anchor)| anchored(*anchor)) + .map(|&(query, _)| query) + .collect() + } + /// Keeps, of the executions answering each query output, those `keep` /// keeps (given the query, the execution's position and the /// execution). @@ -410,6 +426,42 @@ pub(crate) fn has_one_reader_per_context(properties: PhysicalProperties) -> bool properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime } +/// Per memo group of `physical`, each (query, anchor) whose walk from the +/// groups its outputs read (`outputs`) reaches it ([`query_reads`]), +/// through the inputs of every candidate of a group. +fn reads_of_groups( + physical: &PhysicalASAPMemoCandidates, + entries: &[QueryWorkloadEntry], + outputs: &[(QueryId, MemoGroupId)], +) -> HashMap)>> { + let memo = &physical.logical_candidates.memo; + let query_count = outputs + .iter() + .map(|(query, _)| query.0 + 1) + .max() + .unwrap_or(0); + let mut starts = vec![Vec::new(); query_count]; + for &(query, group) in outputs { + starts[query.0].push(group); + } + let inputs_of = |group: MemoGroupId| -> Vec { + let inputs: BTreeSet = memo.groups()[group.0 as usize] + .candidates + .iter() + .flat_map(|candidate| candidate.expression.operator.children()) + .copied() + .collect(); + inputs.into_iter().collect() + }; + query_reads( + &starts, + inputs_of, + |group| request_anchor(&memo.groups()[group.0 as usize].expression), + |_| false, + entries, + ) +} + /// Each query output reference (a query, and the group its operator root, /// or one operator of its scalar root, reads), in order. pub(crate) fn output_groups(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, MemoGroupId)> { diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index ff616346c..8017eeaa1 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -511,7 +511,7 @@ candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `selectable.rs`, | Pass 1 | `pass1_memo` → `Memo`, the one entry: inside it, the computation targets (walking the memo groups, less the summaries the deployment does not support; `ComputationTarget` is Pass 1's private working type), then each target's ways to compute added as candidates of the groups, a summary with one build per configuration of its `parameter_list` | `stage1-pass1-per-computation-candidates/src/lib.rs`; `computation_targets.rs`; `candidates/` (`mod.rs`, `pending.rs`, `measure_output.rs`); `summary_realization.rs` builds the summary inputs (the sketch algorithms that serve an intent, each row's update) | | Memo types | `Memo`, `MemoGroup`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges; then each group's evaluation grid and the groups each query may read only exactly | `stage1-pass2-reuse-candidates/src/` (`lib.rs`, `rules/{candidates.rs,rollup/,topk.rs,chains.rs}`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, `evaluation_grids.rs`; the readout levels: `asap_accuracy::readout_levels`; the groups each query may read only exactly: `asap_accuracy::ErrorMetricsOfMemo::exact_only_groups_of_query`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | -| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `retention.rs` for $h_v$ and (R), `exact_only.rs` for each query's exact-only reads $a_{q,e}$ and (X), `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | +| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `retention.rs` for $h_v$ and (R), `exact_only.rs` for each query's exact-only reads $a_{q,e}$ and (X), `dominance.rs` for executions their `OneEvaluation` twin dominates, `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies From 142949081f83f51e23ab3d7fdf427a24698e6ff4 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:12:02 +0000 Subject: [PATCH 15/23] test(types): an ingestion-time view without a window delivers on every context (#1207 PR 7) --- .../stages/physical/memo_candidates/tests.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 8f8775588..ffd54b47f 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -410,6 +410,36 @@ fn a_count_over_arriving_rows_computed_at_query_time_delivers_on_its_own_grid_on ); } +/// A view kept up to date at ingestion time with no window (SQL +/// `count(*)` over the arriving rows, stored across evaluations) folds in +/// every arriving batch, so it is current at every time: it delivers on +/// every evaluation grid and every ingestion pipeline, whatever its period +/// and phase, not only on its group's 60 s grid. +#[test] +fn an_ingestion_time_view_without_a_window_delivers_on_every_context() { + for arrival in ARRIVING { + let examples = finality_examples(arrival); + let stored = candidate( + examples.count_of_table, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + for context in [ + evaluation(60_000, 0), + evaluation(30_000, 0), + evaluation(45_000, 15_000), + evaluation(7_000, 3_000), + ingestion(1_000, 0), + ingestion(60_000, 30_000), + ] { + assert!( + stored.delivers_on(&examples.candidates, &context), + "{arrival:?} {context:?}" + ); + } + } +} + /// Case (b) of #1264: in `max_over_time((rate(latency[5m] @ end()))[10m:1m] /// @ T)`, `@ T` pins only the subquery's own time; the inner `@ end()` /// still moves with the request. Even over data at rest, a stored result From 70701e90d75403eec6c2c0f900687242463d79fd Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:04:01 +0000 Subject: [PATCH 16/23] refactor(types,cost,stage3): place shared rules where their facts live; one raw-row-store reader rule (#1207 PR 7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - IR: `Operator::chain_tumbling_window_width_ms` and `Operator::request_anchor`; Stage 3, asap-cost and Pass 2 call them. - Logical: `EvaluationGrid`, `LogicalASAPMemoCandidates::own_schedule` (an `EvaluationGrid`) and `query_must_not_read`; `Memo::output_groups`. `PhysicalCandidate` keeps `delivers_on` only. - Physical: `PhysicalProperties::has_one_reader_per_context`, `PhysicalASAPMemoCandidates::operator`, `ExecutionContext::EVERY_ARRIVING_ROW`, and the context rules `queries_of_context`, `anchor_is_evaluation_time` and `query_reads` (moved from asap-cost, which keeps depending on types only). - Workload: `QueryWorkloadEntry::evaluation_grid_ms` and `request_time_grid_ms`. - Plan: `PhysicalASAPDAGNode::keeps_raw_rows_at_ingestion` (a `Scan`, or a `TimeShift` without `@` over one), with `asap_cost::reads_one_batch_of`: an ingestion-time reader of a raw-row store reads one batch, in Stage 3 and now in `plan_cost` too (it read the whole store there); the cost model's raw-row rule uses the same predicate. - asap-cost: `output_interval_ms` is public; Stage 3's Δ of π_v calls it. - Stage 3: `SelectableExecutions::reads_back` and `Reader`; the `is_across_evaluations` wrapper and `is_query_time_scan`'s memo walk replaced by `PhysicalASAPMemoCandidates::operator`. --- crates/cost/src/lib.rs | 4 +- crates/cost/src/node/usage/mod.rs | 86 ++++----- crates/cost/src/plan/cost.rs | 42 ++++- crates/cost/src/plan/execution_demand/mod.rs | 168 ++---------------- .../cost/src/plan/execution_demand/tests.rs | 1 + .../src/evaluation_grids.rs | 12 +- .../src/rules/chains.rs | 13 +- .../src/test_fixtures.rs | 10 +- crates/stage3-plan-selection/src/cost.rs | 76 ++++---- crates/stage3-plan-selection/src/lib.rs | 19 +- .../src/milp/exact_only.rs | 46 ++--- .../src/milp/model/mod.rs | 10 +- .../src/milp/model/tests.rs | 19 +- .../src/milp/retention.rs | 21 ++- .../src/milp/upper_bound.rs | 9 +- crates/stage3-plan-selection/src/plan.rs | 5 +- crates/stage3-plan-selection/src/retention.rs | 62 +------ .../stage3-plan-selection/src/selectable.rs | 141 ++++++++------- crates/types/src/ir/operator/node.rs | 26 ++- .../src/plan/materialized_view/raw_data.rs | 18 ++ crates/types/src/stages/logical/memo/mod.rs | 16 ++ .../src/stages/logical/memo_candidates.rs | 54 +++++- crates/types/src/stages/logical/mod.rs | 2 +- crates/types/src/stages/mod.rs | 12 +- crates/types/src/stages/physical/candidate.rs | 71 +++----- .../src/stages/physical/execution_context.rs | 133 +++++++++++++- .../stages/physical/memo_candidates/mod.rs | 20 ++- .../stages/physical/memo_candidates/tests.rs | 2 +- crates/types/src/stages/physical/mod.rs | 5 +- .../src/stages/physical/properties/mod.rs | 9 + crates/types/src/workload/query.rs | 27 +++ 31 files changed, 610 insertions(+), 529 deletions(-) diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index 86ef8062c..e170a6940 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -40,8 +40,8 @@ pub use model::{ }; pub use node::operator_statistics::EdgeStatistics; pub use node::usage::{ - maintained_window_ms, node_resource_usage_and_cost, summary_type_name, NodeCost, - NodeResourceUsageAndCost, ProducerOutput, + maintained_window_ms, node_resource_usage_and_cost, output_interval_ms, reads_one_batch_of, + summary_type_name, NodeCost, NodeResourceUsageAndCost, ProducerOutput, }; pub use plan::cost::{ plan_cost, raw_data_disk_cost_per_second, request_overhead_cost_per_second, PlanCost, diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 6411a13ee..146c73227 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -22,7 +22,7 @@ use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; use asap_types::plan::physical_asap_dag::{ - PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, + PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; use asap_types::stages::{Lifetime, StorageMedium}; use asap_types::summary_type::{SketchParams, SketchStatistic}; @@ -202,6 +202,22 @@ impl<'a> NodeResourceUsageAndCost<'a> { } } +/// Whether `consumer` reads only one batch of `producer`'s output: it runs +/// at ingestion time and `producer` keeps raw rows stored at ingestion +/// (`PhysicalASAPDAGNode::keeps_raw_rows_at_ingestion`, over +/// `producer_input`, its first producer's operator), whose output is the +/// rows it retains, what a query-time reader reads. Such a reader reads +/// the arriving batch, as from a `Scan` that is not materialized; Stage 3 +/// and [`plan_cost`](crate::plan_cost) both cost it so. +pub fn reads_one_batch_of( + consumer: &PhysicalASAPDAGNode, + producer: &PhysicalASAPDAGNode, + producer_input: Option<&Operator>, +) -> bool { + consumer.output_state.timing == ExecutionTiming::IngestionTime + && producer.keeps_raw_rows_at_ingestion(producer_input) +} + /// One second, in ms: an ingestion-time node runs over the rows arriving /// in one second, and an ingestion-time view that maintains no window is /// refreshed once per second. @@ -225,23 +241,20 @@ pub fn maintained_window_ms<'o, N: Copy + Ord, C: 'o>( input_of: impl Fn(N) -> Option, ) -> Option { let range_ms = |range: &std::time::Duration| u64::try_from(range.as_millis()).ok(); - match operator_of(node) { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => return Some(*tumbling_window_width_ms), - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), - _ => {} + if let Some(width_ms) = operator_of(node).chain_tumbling_window_width_ms() { + return Some(width_ms); + } + if let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = operator_of(node) { + return range_ms(range); } let mut seen = BTreeSet::new(); let mut below = input_of(node)?; while seen.insert(below) { + if let Some(width_ms) = operator_of(below).chain_tumbling_window_width_ms() { + return Some(width_ms); + } match operator_of(below) { Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => return Some(*tumbling_window_width_ms), Operator::NonASAP( NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, ) => below = input_of(below)?, @@ -251,6 +264,18 @@ pub fn maintained_window_ms<'o, N: Copy + Ord, C: 'o>( None } +/// The time between two outputs of a node running `payload` that keeps +/// rows, which its retention is divided by: a chain's tumbling window, +/// else `window_ms` (the window it maintains), else one second of +/// arriving rows; at least 1 ms. Stage 3's $\pi_v$ is the cost of one +/// more of it. +pub fn output_interval_ms(payload: &Operator, window_ms: Option) -> u64 { + payload + .chain_tumbling_window_width_ms() + .unwrap_or_else(|| window_ms.unwrap_or(ONE_SECOND_MS)) + .max(1) +} + /// Measure and cost one node of a physical DAG (one step of /// [`plan_cost`]; Stage 3 calls it per candidate): `producers` are its /// producers in edge order, `rate` its `(multiplier, per_second)` @@ -298,14 +323,7 @@ pub fn node_resource_usage_and_cost<'a>( // windows; any other view materialized at ingestion time has one output // per refresh (per window it maintains, `window_ms`; per second of // arriving rows without one). - let output_interval_ms = match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => *tumbling_window_width_ms, - _ => window_ms.unwrap_or(ONE_SECOND_MS), - } - .max(1); + let output_interval_ms = output_interval_ms(&node.payload, window_ms); // A view materialized across evaluations keeps what its retention // covers. Without one (a reader reads it unbounded: a negative offset, // an `@` anchor) nothing bounds what it keeps and it cannot be @@ -347,18 +365,8 @@ pub fn node_resource_usage_and_cost<'a>( // by an offset over one, across evaluations. It is a time-indexed store // its consumers read back in, retaining `retention_ms` ($h_v$) of rows, // which behind an offset includes the offset. - let raw_rows_materialized_at_ingestion = ingestion - && matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) - && match &node.payload { - Payload::NonASAP(NonASAPOp::Scan { .. }) => true, - Payload::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - shift.at.is_none() - && producers.first().is_some_and(|producer| { - matches!(producer.payload, Payload::NonASAP(NonASAPOp::Scan { .. })) - }) - } - _ => false, - }; + let raw_rows_materialized_at_ingestion = + node.keeps_raw_rows_at_ingestion(producers.first().map(|producer| producer.payload)); // The raw rows of its sources back `lookback_ms`: (by arrival rate, // when known; the rows they hold, when declared). let raw_rows = |lookback_ms: Option| -> (Option, Option) { @@ -1053,14 +1061,12 @@ pub fn node_resource_usage_and_cost<'a>( if cost.is_some_and(|cost| !cost.is_finite()) { return Err(AnalyticalCostError::Overflow); } - let chain_tumbling_window_width_ms = match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(*tumbling_window_width_ms), - Payload::NonASAP(NonASAPOp::TimeShift { .. }) => chain_tumbling_window_width_ms(producers), - _ => None, - }; + let chain_tumbling_window_width_ms = + node.payload.chain_tumbling_window_width_ms().or_else(|| { + matches!(node.payload, Payload::NonASAP(NonASAPOp::TimeShift { .. })) + .then(|| chain_tumbling_window_width_ms(producers)) + .flatten() + }); Ok(NodeResourceUsageAndCost { output: out, sources, diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 473aeec90..dd4049a96 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -11,7 +11,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use crate::node::analytical::AnalyticalCostError; use crate::node::operator_statistics::EdgeStatistics; use crate::node::state_bytes::{sketch_shape, StateInputRows}; -use crate::node::usage::{cost_source, maintained_window_ms}; +use crate::node::usage::{cost_source, maintained_window_ms, reads_one_batch_of}; use crate::plan::execution_demand::{computations, evaluation_rate, query_starts}; use crate::plan::read_back::read_back_ms; use crate::{lookup, StateFamily}; @@ -27,6 +27,7 @@ use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; +use asap_types::stages::Lifetime; use asap_types::summary_type::SketchParams; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; @@ -108,13 +109,38 @@ pub fn plan_cost( for e in &dag.edges { producers_of.entry(e.consumer).or_default().push(e.producer); } + let first_input_of = |id: PhysicalASAPNodeId| { + let input = producers_of.get(&id)?.first()?; + Some(&nodes[input].payload) + }; + // Each node keeping raw rows at ingestion, not stored: what an + // ingestion-time reader of it reads, one batch + // ([`reads_one_batch_of`]). + let one_batch_nodes: HashMap = dag + .nodes + .iter() + .filter(|node| node.keeps_raw_rows_at_ingestion(first_input_of(node.id))) + .map(|node| { + let mut one_batch = node.clone(); + one_batch.lifetime = Lifetime::None; + one_batch.retention_ms = None; + (node.id, one_batch) + }) + .collect(); let mut costed: HashMap> = HashMap::new(); + let mut one_batch: HashMap> = HashMap::new(); for node in &dag.nodes { let views: Vec> = producers_of .get(&node.id) .map_or(&[][..], Vec::as_slice) .iter() - .map(|p| costed[p].producer_output(&nodes[p].payload, &nodes[p].output_schema)) + .map(|p| { + let read = match reads_one_batch_of(node, nodes[p], first_input_of(*p)) { + true => &one_batch[p], + false => &costed[p], + }; + read.producer_output(&nodes[p].payload, &nodes[p].output_schema) + }) .collect(); let window_ms = match node.output_state.timing { ExecutionTiming::IngestionTime => maintained_window_ms( @@ -133,6 +159,18 @@ pub fn plan_cost( settings, ) .map_err(|error| (node.id, error))?; + if let Some(unstored) = one_batch_nodes.get(&node.id) { + let batch = node_resource_usage_and_cost( + unstored, + &views, + multipliers[&node.id], + window_ms, + data, + settings, + ) + .map_err(|error| (node.id, error))?; + one_batch.insert(node.id, batch); + } costed.insert(node.id, node_costed); } let mut output: HashMap = HashMap::new(); diff --git a/crates/cost/src/plan/execution_demand/mod.rs b/crates/cost/src/plan/execution_demand/mod.rs index eaeb77a6e..2004b1a05 100644 --- a/crates/cost/src/plan/execution_demand/mod.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -10,12 +10,10 @@ use crate::node::analytical::AnalyticalCostError; use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::ir::Operator; use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use asap_types::stages::Lifetime; -use asap_types::workload::{ - greatest_common_divisor, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand, -}; +use asap_types::stages::{queries_of_context, query_reads, Lifetime}; +use asap_types::workload::{QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; /// How a [`ScheduleGroup`]'s evaluations add to the others'. @@ -298,89 +296,9 @@ pub enum Consumer { Materialized(PhysicalASAPNodeId), } -/// How each query reads the nodes below it, in a plan or a memo: per node, -/// each (query, anchor) whose walk from the query's `starts` down -/// `inputs_of` reaches it, once, skipping (and stopping at) a node -/// `stops_at` keeps. Below a `TimeShift` anchored with `@ end()` or -/// `@ start()` (`anchor_of`) a query reads per request, with that anchor, -/// unless the anchor is its evaluation time ([`anchor_is_evaluation_time`] -/// over the queries on its grid that reach the shift through any node); -/// elsewhere with none. Stage 3 walks memo groups with it and -/// [`consumers`] plan nodes, so both rate a context by the queries whose -/// walk reaches it. Stage 3 walks every candidate of a group, so a query -/// reading a group through one candidate counts for the group's other -/// candidates too. -/// -/// Example: an instant `sum_over_time(m[5m] @ end())` and a range -/// `sum_over_time(m[5m] @ end() offset 10m)`, both every 60 s: `m[5m]` is -/// read by the instant query with no anchor and by the range query with -/// `@ end()` (its requests' ends are not its evaluations). -pub fn query_reads( - starts: &[Vec], - inputs_of: impl Fn(N) -> Vec, - anchor_of: impl Fn(N) -> Option, - stops_at: impl Fn(N) -> bool, - entries: &[QueryWorkloadEntry], -) -> HashMap)>> { - // The queries reaching each anchored shift, through any node. - let mut reaching_shift: HashMap> = HashMap::new(); - for (query, start) in starts.iter().enumerate() { - let mut seen = HashSet::new(); - let mut stack = start.clone(); - while let Some(id) = stack.pop() { - if seen.insert(id) { - if anchor_of(id).is_some() { - reaching_shift.entry(id).or_default().insert(query); - } - stack.extend(inputs_of(id)); - } - } - } - let in_evaluation_context = |shift: N, query: usize, at: AtModifier| { - let Some(entry) = entries.get(query) else { - return false; - }; - let on_grid = queries_of_context( - evaluation_grid_ms(entry), - reaching_shift.get(&shift).into_iter().flatten().copied(), - entries, - ); - anchor_is_evaluation_time(at, on_grid.iter().map(|&q| &entries[q])) - }; - let mut reads: HashMap)>> = HashMap::new(); - for (query, start) in starts.iter().enumerate() { - let mut seen = HashSet::new(); - let mut stack: Vec<(N, Option)> = start.iter().map(|&id| (id, None)).collect(); - while let Some((id, anchor)) = stack.pop() { - if stops_at(id) || !seen.insert((id, anchor)) { - continue; - } - reads.entry(id).or_default().push((query, anchor)); - let below = match (anchor, anchor_of(id)) { - (None, Some(at)) if !in_evaluation_context(id, query, at) => Some(at), - (anchor, _) => anchor, - }; - stack.extend(inputs_of(id).into_iter().map(|input| (input, below))); - } - } - reads -} - -/// The anchor of `operator` when it is a `TimeShift` anchored with -/// `@ end()` or `@ start()` ([`query_reads`]). -pub fn request_anchor(operator: &Operator) -> Option { - match operator { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { - Some(at @ (AtModifier::End | AtModifier::Start)) => Some(at), - _ => None, - }, - _ => None, - } -} - /// The consumers of each node of `dag` not materialized across evaluations /// that any consumer reaches ([`computations`]): the queries -/// ([`query_reads`]) and the nodes materialized across evaluations whose +/// ([`query_reads`](asap_types::stages::query_reads)) and the nodes materialized across evaluations whose /// walk down their inputs reaches it before meeting such a node, each /// once, sorted. A `OneEvaluation` node is computed within its consumers' /// context, so a walk goes through it. @@ -414,7 +332,7 @@ pub fn consumers( |id| { payload .get(&id) - .and_then(|operator| request_anchor(operator)) + .and_then(|operator| operator.request_anchor()) }, |id| is_materialized.contains(&id), entries, @@ -451,74 +369,6 @@ pub fn consumers( consumers } -/// Whether a `TimeShift` anchored with `at`, read by `queries` (those on -/// one evaluation grid that reach it), is anchored at their evaluation -/// times, so its input runs in their evaluation context, shared with -/// their other reads: `@ end()` with an instant query among them (an -/// instant query's request ends at its evaluation; the input computed at -/// every evaluation of the grid serves the range queries' request ends -/// too). Otherwise it runs once per request, in a context of its own. -/// -/// `@ start()` of an instant query is its evaluation time too; it keeps -/// its request context here, as the user decided for `@ end()` only. -/// -/// Example: `sum_over_time(m[5m] @ end())` and `sum_over_time(m[5m])`, -/// both instant every 60 s: true, so `m[5m]` is computed once per minute -/// for both; the same `@ end()` in a range query alone: false. -pub fn anchor_is_evaluation_time<'e>( - at: AtModifier, - queries: impl IntoIterator, -) -> bool { - at == AtModifier::End - && queries.into_iter().any(|entry| { - entry - .range_evaluation - .is_none_or(|range| range.evaluations_per_request() <= 1) - }) -} - -/// The queries of the execution context on evaluation grid `grid_ms` -/// among `reaching` (the queries reaching a node, by index in `entries`): -/// those evaluated on that grid ([`evaluation_grid_ms`]; 1 ms for a query -/// without an entry). Stage 3 (over the queries reaching a memo group or -/// an anchored shift) and [`computations`] (over the queries whose walk -/// reaches a plan node) pick a context's queries with it. -pub fn queries_of_context( - grid_ms: u64, - reaching: impl IntoIterator, - entries: &[QueryWorkloadEntry], -) -> Vec { - reaching - .into_iter() - .filter(|&query| entries.get(query).map_or(1, evaluation_grid_ms) == grid_ms) - .collect() -} - -/// The grid, in ms, every evaluation of `entry`'s query falls on -/// (`QueryRecurrence::evaluation_grid_ms` of its range evaluation), the -/// grid of its evaluation context; 1 ms when not known, a grid on which -/// only producers stored at every time deliver. -pub fn evaluation_grid_ms(entry: &QueryWorkloadEntry) -> u64 { - entry - .recurrence - .evaluation_grid_ms(entry.range_evaluation) - .unwrap_or(1) -} - -/// The grid, in ms, of the times `entry`'s requests anchor `@ end()` or -/// `@ start()` (`at`) at: its request end times, shifted back by its -/// range-evaluation span for `@ start()`; 1 ms when not known. -/// -/// Example: a query every 60 s over the last hour: `@ end()` 60,000 ms, -/// `@ start()` gcd(60,000, 3,600,000) = 60,000 ms. -pub fn request_time_grid_ms(entry: &QueryWorkloadEntry, at: AtModifier) -> u64 { - let end_ms = entry.recurrence.evaluation_grid_ms(None).unwrap_or(1); - match (at, entry.range_evaluation) { - (AtModifier::Start, Some(range)) => greatest_common_divisor(end_ms, range.span_ms), - _ => end_ms, - } -} - /// How many times per second a node runs in one execution context of the /// queries `queries` (a node not materialized across evaluations, run once /// per execution of the context; Stage 3 and [`computations`] both cost it @@ -587,7 +437,7 @@ pub fn context_executions_per_second<'e>( /// root) or a node materialized across evaluations; a walk stops at the /// first such node it meets, and a node read several times within one /// consumer's walk counts once for it. The queries on one evaluation grid -/// ([`evaluation_grid_ms`]) are one context, and the requests of the +/// ([`QueryWorkloadEntry::evaluation_grid_ms`]) are one context, and the requests of the /// queries on one grid below a `TimeShift` anchored with `@ end()` (or /// `@ start()`) another, each at [`context_executions_per_second`]; a /// materialized node is a context of its own, at its rate. Its @@ -663,7 +513,11 @@ pub fn computations( for (anchor, reaching) in queries_by_anchor { let grids: BTreeSet = reaching .iter() - .map(|&query| entries.get(query).map_or(1, evaluation_grid_ms)) + .map(|&query| { + entries + .get(query) + .map_or(1, QueryWorkloadEntry::evaluation_grid_ms) + }) .collect(); for grid_ms in grids { let queries = queries_of_context(grid_ms, reaching.iter().copied(), entries); diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs index c2fe3f42a..9fba374d6 100644 --- a/crates/cost/src/plan/execution_demand/tests.rs +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -1,4 +1,5 @@ use super::*; +use asap_types::ir::NonASAPOp; use asap_types::workload::{DemandEstimate, Rate, RepetitionInterval, TimestampMs}; fn fixed(interval: u32, phase: u64) -> QueryRecurrence { diff --git a/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs index 1cbb9e09f..bb8265802 100644 --- a/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs +++ b/crates/stage1-pass2-reuse-candidates/src/evaluation_grids.rs @@ -5,7 +5,7 @@ use std::collections::btree_map::Entry; use std::collections::BTreeMap; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::ir::{NonASAPOp, Operator}; use asap_types::stages::{EvaluationGrid, Memo, MemoGroupId}; use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; @@ -92,13 +92,9 @@ pub(crate) fn chain_off_the_evaluation_grid( group: MemoGroupId, input_group: MemoGroupId, ) -> Option<(u64, Option)> { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = memo.groups()[input_group.0 as usize].expression - else { - return None; - }; + let tumbling_window_width_ms = memo.groups()[input_group.0 as usize] + .expression + .chain_tumbling_window_width_ms()?; if !memo.groups()[group.0 as usize] .group_properties .depends_on_evaluation_time diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs index 9784f06af..1e0f9c990 100644 --- a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs +++ b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs @@ -231,14 +231,11 @@ fn chain_path(memo: &Memo, state: MemoGroupId) -> Option { fn existing_chain_widths(memo: &Memo, path: &ChainPath) -> Vec { memo.groups() .iter() - .filter_map(|group| match &group.expression { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) if group.expression == path.chain(*tumbling_window_width_ms) => { - Some(*tumbling_window_width_ms) - } - _ => None, + .filter_map(|group| { + group + .expression + .chain_tumbling_window_width_ms() + .filter(|&width_ms| group.expression == path.chain(width_ms)) }) .collect() } diff --git a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs index 5213cae39..f0e48388d 100644 --- a/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass2-reuse-candidates/src/test_fixtures.rs @@ -128,11 +128,11 @@ pub fn chain_widths_serving(memo: &Memo, state: MemoGroupId) -> BTreeSet { /// `TimeRange` and `TimeShift` groups, if any. pub fn chain_width_below(memo: &Memo, mut group: MemoGroupId) -> Option { loop { - match &memo.groups()[group.0 as usize].expression { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => return Some(*tumbling_window_width_ms), + let expression = &memo.groups()[group.0 as usize].expression; + if let Some(width_ms) = expression.chain_tumbling_window_width_ms() { + return Some(width_ms); + } + match expression { Operator::NonASAP( NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, ) => group = *child, diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index f6da05729..19b1963dc 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -53,11 +53,10 @@ use std::collections::HashMap; use asap_cost::plan::execution_demand::{ context_executions_per_second, executions_per_second as rate_of_materialized, - queries_of_context, }; use asap_cost::{ - maintained_window_ms, node_resource_usage_and_cost, raw_data_disk_cost_per_second, CostInputs, - NodeResourceUsageAndCost, ProducerOutput, + maintained_window_ms, node_resource_usage_and_cost, output_interval_ms, + raw_data_disk_cost_per_second, CostInputs, NodeResourceUsageAndCost, ProducerOutput, }; use asap_types::ir::operator::Source; use asap_types::ir::properties::ExecutionTiming; @@ -66,15 +65,14 @@ use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, }; use asap_types::stages::{ - ExecutionContext, Lifetime, Memo, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, - PhysicalEdgeId, + queries_of_context, ExecutionContext, Lifetime, Memo, MemoGroupId, PhysicalASAPMemoCandidates, + PhysicalCandidateId, PhysicalEdgeId, }; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::plan::candidate_node; use crate::retention::{ edge_retention_need_ms, edge_scan_read_ms, least_need_ms_per_execution, ms_to_seconds, - reads_back, }; use crate::selectable::{Execution, SelectableExecutions}; @@ -187,7 +185,7 @@ pub(crate) fn execution_costs( *most = most.zip(need_ms).map(|(most, need)| most.max(need)); } }; - for (index, _, need_ms) in reads_back(selectable, &retention_need_ms_per_edge, entries) { + for (index, _, need_ms) in selectable.reads_back(&retention_need_ms_per_edge, entries) { read(executions[index].candidate, need_ms); } // How far back a query-time `Scan` reads for the consumer of each edge @@ -353,13 +351,11 @@ pub(crate) fn execution_costs( let Some(retention_ms) = most_read_ms else { continue; }; - let candidate = &physical.physical_candidates[v.0]; - let delta_ms = match candidate.physical_properties.timing() { + let window_ms = match nodes[v.0].output_state.timing { ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), - ExecutionTiming::QueryTime => candidate.chain_tumbling_window_width_ms(physical), - } - .unwrap_or(1_000) - .max(1); + ExecutionTiming::QueryTime => None, + }; + let delta_ms = output_interval_ms(&nodes[v.0].payload, window_ms); let longer = node_retaining(v, Some(retention_ms + delta_ms)); let delta_seconds = ms_to_seconds(delta_ms); let per_retained_second = match physical.physical_candidates[v.0] @@ -445,7 +441,7 @@ pub(crate) fn execution_costs( let &(own, raw_data, per_second) = at_rate[consumer].as_ref().ok()?; let produced = usage[u.0].as_ref()?.as_ref().ok()?; let mut inputs = baseline[c.0].clone()?; - if !reads_one_batch_of(&nodes, c, u) { + if !reads_one_batch_of(physical, &nodes, c, u) { inputs[e.input_index_of_consumer] = produced.producer_output(&nodes[u.0].payload, &nodes[u.0].output_schema); } @@ -486,15 +482,12 @@ pub(crate) fn execution_costs( /// Whether candidate `v` is a `Scan` at query time, which reads raw data /// in each execution. fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { - let candidate = &physical.physical_candidates[v.0]; - candidate.physical_properties.timing() == ExecutionTiming::QueryTime + physical.physical_candidates[v.0] + .physical_properties + .timing() + == ExecutionTiming::QueryTime && matches!( - physical - .logical_candidates - .memo - .candidate(candidate.logical_candidate_id) - .expression - .operator, + physical.operator(v), Operator::NonASAP(NonASAPOp::Scan { .. }) ) } @@ -503,7 +496,7 @@ fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidat /// serves ([`context_executions_per_second`], which `plan_cost` uses too): /// in an evaluation context, for the queries on its grid whose walk over /// the memo reaches the execution's group in their evaluation context -/// (`SelectableExecutions::readers_of`, [`asap_cost::plan::execution_demand::query_reads`], +/// (`SelectableExecutions::readers_of`, [`asap_types::stages::query_reads`], /// the walk `plan_cost` takes over the plan), at their evaluations; for /// the inputs of a `TimeShift` anchored with `@`, once per request of the /// queries on the shift's grid that reach the shift. The memo walk goes @@ -582,9 +575,8 @@ fn baseline_inputs<'a>( usage: &[Option, String>>], v: PhysicalCandidateId, ) -> Option>> { - memo.candidate(physical.physical_candidates[v.0].logical_candidate_id) - .expression - .operator + physical + .operator(v) .children() .into_iter() .enumerate() @@ -596,31 +588,30 @@ fn baseline_inputs<'a>( .admissible_edges_into(v, i) .iter() .map(|&edge| physical.edge(edge).producer_candidate_id) - .filter(|&u| !reads_one_batch_of(nodes, v, u)) + .filter(|&u| !reads_one_batch_of(physical, nodes, v, u)) .filter_map(|u| usage[u.0].as_ref()?.as_ref().ok()), ) }) .collect() } -/// Whether consumer `c` reads only one batch of producer `u`'s output: `c` -/// runs at ingestion time and `u` is a raw `Scan` materialized there, whose -/// output is the rows it retains, what a query-time consumer reads. Such an -/// edge costs what reading a `Scan` that is not materialized does. +/// Whether consumer `c` reads only one batch of producer `u`'s output +/// ([`asap_cost::reads_one_batch_of`], over `u`'s input group's +/// operator): such an edge costs what reading a `Scan` that is not +/// materialized does. fn reads_one_batch_of( + physical: &PhysicalASAPMemoCandidates, nodes: &[PhysicalASAPDAGNode], c: PhysicalCandidateId, u: PhysicalCandidateId, ) -> bool { - let ingestion_time = - |v: PhysicalCandidateId| nodes[v.0].output_state.timing == ExecutionTiming::IngestionTime; - ingestion_time(c) - && ingestion_time(u) - && nodes[u.0].retains_rows() - && matches!( - nodes[u.0].payload, - Operator::NonASAP(NonASAPOp::Scan { .. }) - ) + let memo = &physical.logical_candidates.memo; + let input = physical + .operator(u) + .children() + .first() + .map(|group| &memo.groups()[group.0 as usize].expression); + asap_cost::reads_one_batch_of(&nodes[c.0], &nodes[u.0], input) } /// An input's logical output over its readable costed candidates' `usages`: the most @@ -685,8 +676,9 @@ fn ingestion_window_ms( ) -> Option { let memo = &physical.logical_candidates.memo; let group = physical.physical_candidates[v.0].logical_candidate_id.group; - physical.physical_candidates[v.0] - .chain_tumbling_window_width_ms(physical) + physical + .operator(v) + .chain_tumbling_window_width_ms() .or_else(|| { maintained_window_ms( group, diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 66040b7ac..96d153d83 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -30,15 +30,14 @@ mod selectable; use asap_cost::{request_overhead_cost_per_second, CostInputs, CostModel}; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; -use asap_types::stages::{PhysicalASAPMemoCandidates, PruneReason, PrunedCandidate}; +use asap_types::stages::{MemoGroupId, PhysicalASAPMemoCandidates, PruneReason, PrunedCandidate}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::cost::execution_costs; -use crate::milp::exact_only::must_not_read; use crate::milp::model::{solve_selection, SelectionInputs, SelectionMilpError}; use crate::plan::plan_of; use crate::report::{accepted_accuracy, infeasible}; -use crate::selectable::{output_groups, SelectableExecutions}; +use crate::selectable::{QueryId, SelectableExecutions}; pub use report::{ large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, @@ -77,7 +76,13 @@ pub fn select_plan( entries: entries.len(), }); } - let outputs = output_groups(physical); + let outputs: Vec<(QueryId, MemoGroupId)> = physical + .logical_candidates + .memo + .output_groups() + .into_iter() + .map(|(query, group)| (QueryId(query), group)) + .collect(); let mut selectable = SelectableExecutions::new(physical, entries, &outputs); let request_overhead_cost_per_second = request_overhead_cost_per_second(entries, cost_inputs) .map_err(|error| ErrorSelectingPlan::Cost { @@ -89,7 +94,11 @@ pub fn select_plan( // A query output is answered by an execution that could be costed and // that its query may read (X). selectable.retain_answering(|query, index, execution| { - costs.per_execution[index].is_ok() && !must_not_read(physical, query, execution.candidate) + costs.per_execution[index].is_ok() + && !physical.logical_candidates.query_must_not_read( + query.0, + physical.physical_candidates[execution.candidate.0].logical_candidate_id, + ) }); if selectable .answering() diff --git a/crates/stage3-plan-selection/src/milp/exact_only.rs b/crates/stage3-plan-selection/src/milp/exact_only.rs index a60413329..88b176b15 100644 --- a/crates/stage3-plan-selection/src/milp/exact_only.rs +++ b/crates/stage3-plan-selection/src/milp/exact_only.rs @@ -13,42 +13,14 @@ use std::collections::{BTreeMap, BTreeSet}; use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; -use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId}; - use super::model::SelectionInputs; use crate::selectable::QueryId; -/// Whether query `query` must not read candidate `v`: it is not exact (it -/// has sketch parameters θ) and its group is one the query may read only -/// exactly. A readout's own candidate has no θ, but the sketch states it -/// reads are in the same list, as every group below a listed one is. -/// -/// Example: `quantile_over_time(0.9, m[5m])` at a relative target lists -/// the quantile and every group below it; its KLL state builds (θ) are -/// candidates it must not read, the exact quantile one it may. -pub(crate) fn must_not_read( - physical: &PhysicalASAPMemoCandidates, - query: QueryId, - v: PhysicalCandidateId, -) -> bool { - let logical = &physical.logical_candidates; - let id = physical.physical_candidates[v.0].logical_candidate_id; - logical - .exact_only_groups_of_query - .get(query.0) - .is_some_and(|groups| groups.contains(&id.group)) - && logical - .memo - .candidate(id) - .logical_properties - .parameters() - .is_some() -} - /// $a_{q,e}$, added to `problem`, for each query $q$ and each execution $e$ /// (by position in `SelectableExecutions::executions`) it may reach from /// its output executions over selectable edges (`y`) from which an -/// execution it must not read ([`must_not_read`]) is reachable, less those +/// execution it must not read +/// (`LogicalASAPMemoCandidates::query_must_not_read`) is reachable, less those /// themselves. None for a query with no exact-only group. pub(crate) fn exact_only_variables( inputs: &SelectionInputs<'_>, @@ -74,7 +46,11 @@ pub(crate) fn exact_only_variables( .map(|&position| edges[position].consumer) }; let forbidden = |query: QueryId, e: usize| { - must_not_read(inputs.physical, query, selectable.executions()[e].candidate) + let v = selectable.executions()[e].candidate; + inputs.physical.logical_candidates.query_must_not_read( + query.0, + inputs.physical.physical_candidates[v.0].logical_candidate_id, + ) }; let queries: BTreeSet = selectable.answering().iter().map(|(q, _)| *q).collect(); let mut tracked = BTreeMap::new(); @@ -111,7 +87,8 @@ pub(crate) fn exact_only_variables( /// candidates: $a_{q,e} \ge z_{q,e}$ for an output execution, /// $a_{q,p} \ge y_{\kappa,p,c,i} + a_{q,c} - 1$ along a selectable edge, so /// $a_{q,e} = 1$ on every execution $q$'s selected path reads; along an -/// edge into an execution $p$ $q$ must not read ([`must_not_read`]), +/// edge into an execution $p$ $q$ must not read +/// (`LogicalASAPMemoCandidates::query_must_not_read`), /// $y_{\kappa,p,c,i} + a_{q,c} \le 1$. No $z_{q,e}$ exists for an output /// execution $q$ must not read (`crate::select_plan` leaves it out). Other /// queries may still read it, through their own paths. $a$ exists only @@ -137,7 +114,10 @@ pub(crate) fn exact_only_constraints( let (consumer, producer) = (edge.consumer, edge.producer); for &(query, a_consumer) in readers.get(&consumer).into_iter().flatten() { let v = selectable.executions()[producer].candidate; - if must_not_read(inputs.physical, query, v) { + if inputs.physical.logical_candidates.query_must_not_read( + query.0, + inputs.physical.physical_candidates[v.0].logical_candidate_id, + ) { constraints.push(constraint!(y + a_consumer <= 1)); } else if let Some(&a_producer) = a.get(&(query, producer)) { constraints.push(constraint!(a_producer >= y + a_consumer - 1)); diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 93586f602..33a52f12c 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -34,7 +34,7 @@ use super::graph::Components; use super::retention::{least_retention_ms, retention_constraints, retention_variables}; use super::upper_bound::dearer_than_a_feasible_plan; use crate::cost::ExecutionCosts; -use crate::selectable::{has_one_reader_per_context, Execution, QueryId, SelectableExecutions}; +use crate::selectable::{Execution, QueryId, SelectableExecutions}; /// What the MILP is built over. pub(crate) struct SelectionInputs<'a> { @@ -197,13 +197,15 @@ fn query_coverage_constraints( } /// Whether the execution at position `index` has one reader per context -/// ([`has_one_reader_per_context`]): its (F) row is the equality +/// (`PhysicalProperties::has_one_reader_per_context`): its (F) row is the equality /// $\sum y + \sum_q z_{q,v} = x_{\kappa,v}$ ([`use_constraints`]), which /// with $y, z \ge 0$ implies (B) and (D) for it, so those rows are left /// out. pub(super) fn has_one_reader(inputs: &SelectionInputs<'_>, index: usize) -> bool { let v = inputs.selectable.executions()[index].candidate; - has_one_reader_per_context(inputs.physical.physical_candidates[v.0].physical_properties) + inputs.physical.physical_candidates[v.0] + .physical_properties + .has_one_reader_per_context() } /// (E) $t_u + 1 \le t_c + n(1 - y_{\kappa,u,c,i})$ between the producer's @@ -232,7 +234,7 @@ fn acyclicity_constraints( /// (F) $x_{\kappa,v} \le \sum y + \sum_q z_{q,v}$ over the edges and query /// outputs reading that execution; with it, a candidate with one reader -/// per context ([`has_one_reader_per_context`]: a query-time result that +/// per context (`PhysicalProperties::has_one_reader_per_context`: a query-time result that /// is not materialized) has $\sum y + \sum_q z_{q,v} \le x_{\kappa,v}$, so /// for such an execution the two are one row, $\sum y + \sum_q z_{q,v} = /// x_{\kappa,v}$. An ingestion-time producer is exempt from the diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index c67b900a9..68607ac14 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -28,10 +28,9 @@ use asap_types::workload::{ use super::*; use crate::cost::ExecutionCosts; -use crate::milp::exact_only::must_not_read; use crate::plan::plan_of; use crate::retention::{edge_retention_need_ms, least_need_ms_per_execution}; -use crate::selectable::{output_groups, SelectableEdge}; +use crate::selectable::{QueryId, SelectableEdge}; const NONE: Lifetime = Lifetime::None; const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); @@ -149,7 +148,10 @@ fn selection_inputs( ) -> (SelectableExecutions, ExecutionCosts) { let mut selectable = SelectableExecutions::new(physical, entries, &output_groups(physical)); selectable.retain_answering(|query, _, execution| { - !must_not_read(physical, query, execution.candidate) + !physical.logical_candidates.query_must_not_read( + query.0, + physical.physical_candidates[execution.candidate.0].logical_candidate_id, + ) }); let retention_cost_per_retained_second: HashMap = retention .iter() @@ -223,6 +225,17 @@ fn solve( (solved, selectable) } +/// Each query output reference of `physical`'s memo (`Memo::output_groups`). +fn output_groups(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, MemoGroupId)> { + physical + .logical_candidates + .memo + .output_groups() + .into_iter() + .map(|(query, group)| (QueryId(query), group)) + .collect() +} + /// The candidates of `solved`'s executions, one per execution, ascending. fn candidates(solved: &SolvedSelection) -> Vec { let mut candidates: Vec = solved diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs index faca6e1ab..6f53a03ce 100644 --- a/crates/stage3-plan-selection/src/milp/retention.rs +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -11,8 +11,9 @@ use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; use asap_types::stages::PhysicalCandidateId; use super::model::SelectionInputs; -use crate::retention::{ms_to_seconds, reads_back, Reader}; +use crate::retention::ms_to_seconds; use crate::selectable::QueryId; +use crate::selectable::Reader; /// $h_v$ in seconds retained (the unit keeps (R)'s coefficients near the /// others'), added to `problem`, for each candidate with a retention cost @@ -66,11 +67,10 @@ pub(crate) fn retention_constraints( ) -> Vec { let executions = inputs.selectable.executions(); let mut constraints = Vec::new(); - for (index, reader, need_ms) in reads_back( - inputs.selectable, - &inputs.costs.retention_need_ms_per_edge, - inputs.entries, - ) { + for (index, reader, need_ms) in inputs + .selectable + .reads_back(&inputs.costs.retention_need_ms_per_edge, inputs.entries) + { let Some(&h) = h.get(&executions[index].candidate) else { continue; }; @@ -108,11 +108,10 @@ pub(crate) fn least_retention_ms( let edges: HashSet = edges.into_iter().collect(); let outputs: HashSet<(QueryId, usize)> = outputs.into_iter().collect(); let mut retention_ms: HashMap = HashMap::new(); - for (index, reader, need_ms) in reads_back( - inputs.selectable, - &inputs.costs.retention_need_ms_per_edge, - inputs.entries, - ) { + for (index, reader, need_ms) in inputs + .selectable + .reads_back(&inputs.costs.retention_need_ms_per_edge, inputs.entries) + { let selected = match reader { Reader::Edge(position) => edges.contains(&position), Reader::Output(query) => outputs.contains(&(query, index)), diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index 6536fbf5c..a36f90842 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -14,7 +14,6 @@ use std::collections::{BTreeMap, HashMap}; -use super::exact_only::must_not_read; use super::model::{has_one_reader, selection_cost_per_second, SelectionInputs}; use crate::selectable::QueryId; @@ -94,8 +93,12 @@ fn feasible_plan_cost_per_second(inputs: &SelectionInputs<'_>) -> Option { let one_reader: Vec = (0..count) .map(|index| has_one_reader(inputs, index)) .collect(); - let forbidden = - |query: QueryId, e: usize| must_not_read(inputs.physical, query, executions[e].candidate); + let forbidden = |query: QueryId, e: usize| { + inputs.physical.logical_candidates.query_must_not_read( + query.0, + inputs.physical.physical_candidates[executions[e].candidate.0].logical_candidate_id, + ) + }; // Per query, the executions its trees leave out: those it must not // read, and those a repair left out for it. let mut excluded: BTreeMap> = BTreeMap::new(); diff --git a/crates/stage3-plan-selection/src/plan.rs b/crates/stage3-plan-selection/src/plan.rs index 466ede909..658a6a408 100644 --- a/crates/stage3-plan-selection/src/plan.rs +++ b/crates/stage3-plan-selection/src/plan.rs @@ -34,9 +34,8 @@ pub(crate) fn candidate_node( let logical = memo.candidate(candidate.logical_candidate_id); let group = &memo.groups()[candidate.logical_candidate_id.group.0 as usize].group_properties; let mut inputs = input_ids.iter(); - let payload = logical - .expression - .operator + let payload = physical + .operator(v) .map_children(|_| *inputs.next().expect("one node per input")); let lifetime = candidate.physical_properties.lifetime(); PhysicalASAPDAGNode { diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index d965d80dd..e4328ed28 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -4,7 +4,7 @@ //! or for a query reading it directly, and how far back one execution of a //! query-time `Scan` reads for that reader: [`asap_cost::read_back_ms`] //! over candidates; and every read of an execution's rows -//! ([`reads_back`]) +//! (`SelectableExecutions::reads_back`) //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 5, "Cost and retention"). @@ -15,7 +15,7 @@ use asap_types::stages::{ }; use asap_types::workload::QueryWorkloadEntry; -use crate::selectable::{QueryId, SelectableExecutions}; +use crate::selectable::{Reader, SelectableExecutions}; /// $\operatorname{need}_{\kappa,u,c,i}$: how far back from the evaluation /// time `edge`'s producer $u$ must keep its rows for its consumer $c$, with @@ -53,54 +53,8 @@ pub(crate) fn edge_scan_read_ms( read_back(physical, edge, |_| 0) } -/// How far back a query reads the output it reads directly: its -/// range-evaluation span (0 without one). -fn output_retention_need_ms(entry: &QueryWorkloadEntry) -> u64 { - entry.range_evaluation.map_or(0, |range| range.span_ms) -} - -/// What reads an execution's rows back ([`reads_back`]): a selectable edge -/// out of it, by position in [`SelectableExecutions::edges`], or a query -/// output it may answer. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Reader { - Edge(usize), - Output(QueryId), -} - -/// Every read of an execution's rows, as (the execution read, by position -/// in [`SelectableExecutions::executions`]; its reader; how far back it -/// reads, in ms, `None` when unbounded): each selectable edge into its -/// producer, `need_ms_per_edge` by edge position -/// ([`edge_retention_need_ms`]), and each query output into each execution -/// that may answer it, the query's range-evaluation span. Costing ($h_v$ -/// at its largest), (R), the least $h_v$ of a selection and the greedy -/// bound each fold it. -pub(crate) fn reads_back<'a>( - selectable: &'a SelectableExecutions, - need_ms_per_edge: &'a [Option], - entries: &'a [QueryWorkloadEntry], -) -> impl Iterator)> + 'a { - let edges = selectable - .edges() - .iter() - .zip(need_ms_per_edge) - .enumerate() - .map(|(position, (edge, &need_ms))| (edge.producer, Reader::Edge(position), need_ms)); - let outputs = selectable - .answering() - .iter() - .flat_map(move |(query, answering)| { - let need_ms = output_retention_need_ms(&entries[query.0]); - answering - .iter() - .map(move |&index| (index, Reader::Output(*query), Some(need_ms))) - }); - edges.chain(outputs) -} - /// The least need, in ms, over each execution's costed reads -/// ([`reads_back`]: an edge out of it `per_edge` costs, or a query output +/// (`SelectableExecutions::reads_back`: an edge out of it `per_edge` costs, or a query output /// it may answer), by position in [`SelectableExecutions::executions`]; /// `None` without a bounded one. (F) gives a selected execution a use and /// (R) then retains at least that: the greedy bound charges it, and the @@ -112,7 +66,7 @@ pub(crate) fn least_need_ms_per_execution( entries: &[QueryWorkloadEntry], ) -> Vec> { let mut least_need_ms = vec![None; selectable.executions().len()]; - for (index, reader, need_ms) in reads_back(selectable, need_ms_per_edge, entries) { + for (index, reader, need_ms) in selectable.reads_back(need_ms_per_edge, entries) { let costed = match reader { Reader::Edge(position) => per_edge[position].is_some(), Reader::Output(_) => true, @@ -198,12 +152,6 @@ impl TimeIndexedPlan for Candidates<'_> { } fn operator_of(&self, v: PhysicalCandidateId) -> &Operator { - &self - .physical - .logical_candidates - .memo - .candidate(self.physical.physical_candidates[v.0].logical_candidate_id) - .expression - .operator + self.physical.operator(v) } } diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index 29a24844f..3a3b36c01 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -12,7 +12,7 @@ //! that selects it ($x_{\kappa,v}$), the context of its consumer; at //! ingestion time, where each arriving row is computed once whatever //! reads it (`asap_cost::computations`), in the one ingestion context -//! [`EVERY_ARRIVING_ROW`]. +//! `ExecutionContext::EVERY_ARRIVING_ROW`. //! - A candidate with $\ell$ = `AcrossEvaluations` runs once, in its own //! context `Materialized(v)` ($x^{\mathrm{mat}}_v = x_{\mathrm{Materialized}(v),v}$). //! Its inputs run in that context at query time; at ingestion time its @@ -24,25 +24,20 @@ //! its end and an evaluation are different times, so their results are //! not shared. An `@ end()` read by an instant query is anchored at its //! evaluation time, so its input runs in the evaluation context -//! (`asap_cost::plan::execution_demand::anchor_is_evaluation_time`). +//! (`asap_types::stages::anchor_is_evaluation_time`). //! - An input of a consumer in $\kappa$ reads a producer, over an admissible //! edge, only where the producer delivers on $\kappa$ //! (`PhysicalCandidate::delivers_on`). use std::collections::{BTreeSet, HashMap}; -use asap_cost::plan::execution_demand::{ - anchor_is_evaluation_time, evaluation_grid_ms, queries_of_context, query_reads, request_anchor, - request_time_grid_ms, -}; use asap_types::ir::operator::AtModifier; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; use asap_types::plan::materialized_view::RefreshWindow; use asap_types::stages::{ - output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, - ExecutionContextId, Lifetime, MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, - PhysicalEdgeId, PhysicalProperties, + anchor_is_evaluation_time, producer_satisfies_physical_requirements, queries_of_context, + query_reads, EvaluationGrid, ExecutionContext, ExecutionContextId, MemoGroupId, + PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, }; use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; @@ -50,14 +45,20 @@ use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueryId(pub usize); -/// The one context of every ingestion-time execution that is not -/// materialized: each second of arriving rows, which it computes once for -/// every ingestion-time view of every refresh window that reads it. -pub(crate) const EVERY_ARRIVING_ROW: ExecutionContext = - ExecutionContext::IngestionPipeline(RefreshWindow { - width_ms: 1_000, - phase_ms: 0, - }); +/// How far back a query reads the output it reads directly: its +/// range-evaluation span (0 without one). +fn output_retention_need_ms(entry: &QueryWorkloadEntry) -> u64 { + entry.range_evaluation.map_or(0, |range| range.span_ms) +} + +/// What reads an execution's rows back ([`SelectableExecutions::reads_back`]): a selectable edge +/// out of it, by position in [`SelectableExecutions::edges`], or a query +/// output it may answer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Reader { + Edge(usize), + Output(QueryId), +} /// One run of candidate `candidate` in context `context`: a node of the /// plan. $x^{\mathrm{mat}}_v$ is the execution of $v$ in @@ -115,7 +116,7 @@ pub(crate) struct SelectableExecutions { edges_out_of: Vec>, /// Each query output reference (a query, and the group its operator /// root, or one operator of its scalar root, reads), in - /// [`output_groups`] order, with the executions, by position, that may + /// `Memo::output_groups` order, with the executions, by position, that may /// answer it: $z_{q,v}$ exists only for them. answering: Vec<(QueryId, Vec)>, } @@ -123,7 +124,7 @@ pub(crate) struct SelectableExecutions { impl SelectableExecutions { /// The contexts, executions and edges of `physical` for the queries /// `entries` (by query index), reached from `outputs` (each query output - /// reference, a query and the group it reads, [`output_groups`]), and + /// reference, a query and the group it reads, `Memo::output_groups`), and /// per output the executions that may answer it: a candidate of its /// group a query-time consumer may read /// (`producer_satisfies_physical_requirements`) that delivers on the @@ -245,7 +246,7 @@ impl SelectableExecutions { /// The execution of producer `u` a consumer whose inputs run in /// `context` reads, when `u` delivers on it: its own for an /// `AcrossEvaluations` producer, the one ingestion context - /// [`EVERY_ARRIVING_ROW`] for another ingestion-time producer, otherwise + /// `ExecutionContext::EVERY_ARRIVING_ROW` for another ingestion-time producer, otherwise /// its run in `context`. fn execution_reading( &mut self, @@ -261,7 +262,7 @@ impl SelectableExecutions { let context = if properties.is_across_evaluations() { self.context_id(ExecutionContext::Materialized(u), None) } else if properties.timing() == ExecutionTiming::IngestionTime { - self.context_id(EVERY_ARRIVING_ROW, None) + self.context_id(ExecutionContext::EVERY_ARRIVING_ROW, None) } else { context }; @@ -273,7 +274,7 @@ impl SelectableExecutions { /// The context the inputs of `consumer` run in: an `AcrossEvaluations` /// candidate's at ingestion time, the ingestion pipeline of its refresh - /// window (`PhysicalCandidate::own_schedule`); a `TimeShift` anchored at + /// window (`LogicalASAPMemoCandidates::own_schedule`); a `TimeShift` anchored at /// `@ end()` or `@ start()` in an evaluation context, a context of its /// own per anchor, on the grid of its queries' request end (start) /// times: a request's start and end, and an evaluation, are different @@ -286,30 +287,29 @@ impl SelectableExecutions { consumer: Execution, ) -> ExecutionContextId { let candidate = &physical.physical_candidates[consumer.candidate.0]; - let memo = &physical.logical_candidates.memo; let group = candidate.logical_candidate_id.group; let properties = candidate.physical_properties; if properties.timing() == ExecutionTiming::IngestionTime && properties.is_across_evaluations() { - return match candidate.own_schedule(physical) { - Some(window) => self.context_id(ExecutionContext::IngestionPipeline(window), None), + return match physical + .logical_candidates + .own_schedule(candidate.logical_candidate_id) + { + Some(grid) => self.context_id( + ExecutionContext::IngestionPipeline(RefreshWindow { + width_ms: grid.period_ms, + phase_ms: grid.phase_ms, + }), + None, + ), None => consumer.context, }; } - let at = match memo - .candidate(candidate.logical_candidate_id) - .expression - .operator - { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at, - _ => None, - }; - let ( - Some(at @ (AtModifier::End | AtModifier::Start)), - ExecutionContext::Evaluation { grid }, - ) = (at, self.contexts[consumer.context.0]) - else { + let (Some(at), ExecutionContext::Evaluation { grid }) = ( + physical.operator(consumer.candidate).request_anchor(), + self.contexts[consumer.context.0], + ) else { return consumer.context; }; let queries: Vec = @@ -321,7 +321,7 @@ impl SelectableExecutions { return consumer.context; } let period_ms = queries.iter().fold(0, |period_ms, query| { - greatest_common_divisor(period_ms, request_time_grid_ms(&entries[query.0], at)) + greatest_common_divisor(period_ms, entries[query.0].request_time_grid_ms(at)) }); match period_ms { 0 => consumer.context, @@ -400,11 +400,37 @@ impl SelectableExecutions { } /// Each query output reference with the executions, by position, that - /// may answer it, in [`output_groups`] order. + /// may answer it, in `Memo::output_groups` order. pub(crate) fn answering(&self) -> &[(QueryId, Vec)] { &self.answering } + /// Every read of an execution's rows, as (the execution read, by + /// position in [`Self::executions`]; its reader; how far back it reads, + /// in ms, `None` when unbounded): each selectable edge into its + /// producer, `need_ms_per_edge` by edge position + /// (`crate::retention::edge_retention_need_ms`), and each query output + /// into each execution that may answer it, the query's range-evaluation + /// span. Costing ($h_v$ at its largest), (R), the least $h_v$ of a + /// selection and the greedy bound each fold it. + pub(crate) fn reads_back<'a>( + &'a self, + need_ms_per_edge: &'a [Option], + entries: &'a [QueryWorkloadEntry], + ) -> impl Iterator)> + 'a { + let edges = + self.edges().iter().zip(need_ms_per_edge).enumerate().map( + |(position, (edge, &need_ms))| (edge.producer, Reader::Edge(position), need_ms), + ); + let outputs = self.answering().iter().flat_map(move |(query, answering)| { + let need_ms = output_retention_need_ms(&entries[query.0]); + answering + .iter() + .map(move |&index| (index, Reader::Output(*query), Some(need_ms))) + }); + edges.chain(outputs) + } + /// The positions in [`Self::edges`] of the edges into input `input` of /// the execution at position `consumer`. pub(crate) fn edges_into(&self, consumer: usize, input: usize) -> &[usize] { @@ -417,15 +443,6 @@ impl SelectableExecutions { } } -/// Whether a candidate with `properties` has at most one consumer per -/// context: a query-time result that is not materialized ($\ell$ = None). -/// Consumers in one context that share a result read a `OneEvaluation` or a -/// materialized producer; the consumers of one ingestion run share its -/// rows. -pub(crate) fn has_one_reader_per_context(properties: PhysicalProperties) -> bool { - properties.lifetime() == Lifetime::None && properties.timing() == ExecutionTiming::QueryTime -} - /// Per memo group of `physical`, each (query, anchor) whose walk from the /// groups its outputs read (`outputs`) reaches it ([`query_reads`]), /// through the inputs of every candidate of a group. @@ -456,35 +473,17 @@ fn reads_of_groups( query_reads( &starts, inputs_of, - |group| request_anchor(&memo.groups()[group.0 as usize].expression), + |group| memo.groups()[group.0 as usize].expression.request_anchor(), |_| false, entries, ) } -/// Each query output reference (a query, and the group its operator root, -/// or one operator of its scalar root, reads), in order. -pub(crate) fn output_groups(physical: &PhysicalASAPMemoCandidates) -> Vec<(QueryId, MemoGroupId)> { - let roots: Vec> = physical - .logical_candidates - .memo - .queries() - .iter() - .map(|query| query.root.clone()) - .collect(); - output_references(&roots) - .into_iter() - .map(|(query, group)| (QueryId(query), group)) - .collect::>() - .into_iter() - .collect() -} - /// The evaluation grid of `entry`'s query, the grid of its context -/// ([`evaluation_grid_ms`]), anchored at the epoch. +/// (`QueryWorkloadEntry::evaluation_grid_ms`), anchored at the epoch. pub(crate) fn evaluation_grid(entry: &QueryWorkloadEntry) -> EvaluationGrid { EvaluationGrid { - period_ms: evaluation_grid_ms(entry), + period_ms: entry.evaluation_grid_ms(), phase_ms: 0, } } diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 19910108e..23bcc7699 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -10,7 +10,7 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; -use super::arguments::Source; +use super::arguments::{AtModifier, Source}; use super::asap::ASAPOp; use super::input::OperatorInput; use super::non_asap::NonASAPOp; @@ -60,6 +60,30 @@ impl Operator { Operator::ASAP(op) => op.kind_name(), } } + + /// Its tumbling-window width when it is a `SummaryTumblingWindowChain`. + pub fn chain_tumbling_window_width_ms(&self) -> Option { + match self { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(*tumbling_window_width_ms), + _ => None, + } + } + + /// The anchor of a `TimeShift` with `@ end()` or `@ start()`, whose + /// input runs once per request; `None` for any other operator (`@ ` + /// included). + pub fn request_anchor(&self) -> Option { + match self { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { + Some(at @ (AtModifier::End | AtModifier::Start)) => Some(at), + _ => None, + }, + _ => None, + } + } } /// The derivation rules, one set for every kind of input diff --git a/crates/types/src/plan/materialized_view/raw_data.rs b/crates/types/src/plan/materialized_view/raw_data.rs index 4ac4be474..707899c13 100644 --- a/crates/types/src/plan/materialized_view/raw_data.rs +++ b/crates/types/src/plan/materialized_view/raw_data.rs @@ -96,6 +96,24 @@ impl PhysicalASAPDAGNode { pub fn is_view(&self) -> bool { matches!(self.lifetime, Lifetime::AcrossEvaluations(_)) && !is_raw_data_scan(self) } + + /// Whether it keeps raw rows stored at ingestion time across + /// evaluations: a `Scan`, or a `TimeShift` without `@` whose input + /// (`input`, its first producer's operator) is a `Scan`. Its readers + /// read the rows it retains; an ingestion-time reader reads one batch + /// of them (`asap_cost::reads_one_batch_of`). + pub fn keeps_raw_rows_at_ingestion(&self, input: Option<&Operator>) -> bool { + self.output_state.timing == ExecutionTiming::IngestionTime + && matches!(self.lifetime, Lifetime::AcrossEvaluations(_)) + && match &self.payload { + Operator::NonASAP(NonASAPOp::Scan { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + shift.at.is_none() + && matches!(input, Some(Operator::NonASAP(NonASAPOp::Scan { .. }))) + } + _ => false, + } + } } impl ViewIndex<'_> { diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 7c492061e..7d733fa2a 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -177,6 +177,22 @@ impl Memo { &self.queries } + /// Each query output reference (the query's index, and the group its + /// operator root, or one operator of its scalar root, reads), + /// deduplicated, ascending. + pub fn output_groups(&self) -> Vec<(usize, MemoGroupId)> { + let roots: Vec> = self + .queries + .iter() + .map(|query| query.root.clone()) + .collect(); + crate::stages::output_references(&roots) + .into_iter() + .collect::>() + .into_iter() + .collect() + } + /// Adds `query` and returns its index: its root's groups and every /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 /// adds each query once. diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index bce6930d6..5bb674155 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -4,7 +4,16 @@ use std::collections::BTreeSet; -use crate::stages::{EvaluationGrid, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate}; +use crate::stages::{ + CandidateForMemoGroupId, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate, +}; + +/// The evaluation times `phase_ms + k · period_ms`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EvaluationGrid { + pub period_ms: u64, + pub phase_ms: u64, +} /// Stage 1, Pass 2's output. #[derive(Debug, Clone)] @@ -41,3 +50,46 @@ pub struct LogicalASAPMemoCandidates { /// target. pub exact_only_groups_of_query: Vec>, } + +impl LogicalASAPMemoCandidates { + /// Whether query `query` must not read `candidate`: it is not exact + /// (it has sketch parameters θ) and its group is one the query may read + /// only exactly ([`Self::exact_only_groups_of_query`]). A readout's own + /// candidate has no θ, but the sketch states it reads are in the same + /// list, as every group below a listed one is. + /// + /// Example: `quantile_over_time(0.9, m[5m])` at a relative target lists + /// the quantile and every group below it; its KLL state builds (θ) are + /// candidates it must not read, the exact quantile one it may. + pub fn query_must_not_read(&self, query: usize, candidate: CandidateForMemoGroupId) -> bool { + self.exact_only_groups_of_query + .get(query) + .is_some_and(|groups| groups.contains(&candidate.group)) + && self + .memo + .candidate(candidate) + .logical_properties + .parameters() + .is_some() + } + + /// The times `candidate` is stored at when materialized across + /// evaluations, on its own schedule: every window end of a chain (its + /// width at phase 0), else its group's evaluation grid + /// ([`Self::evaluation_grid_of_group`]); `None` when neither is known. + pub fn own_schedule(&self, candidate: CandidateForMemoGroupId) -> Option { + match self + .memo + .candidate(candidate) + .expression + .operator + .chain_tumbling_window_width_ms() + { + Some(width_ms) => Some(EvaluationGrid { + period_ms: width_ms, + phase_ms: 0, + }), + None => self.evaluation_grid_of_group[candidate.group.0 as usize], + } + } +} diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index fe018bb8d..a122a1e55 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -26,5 +26,5 @@ pub use group::{ GroupProperties, MemoGroup, MemoGroupError, MemoGroupId, ValueSign, }; pub use memo::{Memo, MemoQuery}; -pub use memo_candidates::LogicalASAPMemoCandidates; +pub use memo_candidates::{EvaluationGrid, LogicalASAPMemoCandidates}; pub use requirements::{LogicalRequirements, RequiredAccuracy}; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 1c70ea118..4e0e91ad1 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -16,16 +16,16 @@ mod pruning; pub use logical::{ listed_build_parameters, ranked_column_of_group, AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, - ColumnLineage, CoverageError, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, - LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, + ColumnLineage, CoverageError, EvaluationGrid, GroupProperties, LogicalASAPMemoCandidates, + LogicalEdge, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupError, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, }; pub use physical::{ - output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, - ExecutionContextId, Lifetime, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PhysicalProperties, PhysicalPropertiesError, - StorageMedium, + anchor_is_evaluation_time, output_references, producer_satisfies_physical_requirements, + queries_of_context, query_reads, ExecutionContext, ExecutionContextId, Lifetime, + PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, + PhysicalEdgeId, PhysicalProperties, PhysicalPropertiesError, StorageMedium, }; pub use pruning::{ PruneCause, PruneReason, PrunedCandidate, PrunedLogicalCandidate, RootPruneReason, diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index f4239de2e..1a06cdfcc 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -2,8 +2,6 @@ //! valid $p^{\mathrm{phys}} = (\tau, \ell)$. use crate::ir::properties::ExecutionTiming; -use crate::ir::{ASAPOp, Operator}; -use crate::plan::materialized_view::RefreshWindow; use crate::stages::{CandidateForMemoGroupId, ExecutionContext, PhysicalASAPMemoCandidates}; use crate::stages::{Lifetime, PhysicalProperties}; use crate::workload::DataArrival; @@ -51,7 +49,8 @@ impl PhysicalCandidate { /// /// `DataArrival::Unknown` counts as arriving. /// The times of a - /// `Materialized(v)` context are $v$'s own ([`Self::own_schedule`]); + /// `Materialized(v)` context are $v$'s own + /// ([`LogicalASAPMemoCandidates::own_schedule`](crate::stages::LogicalASAPMemoCandidates::own_schedule)); /// when they are not known, no non-final producer delivers on it. pub fn delivers_on( &self, @@ -69,7 +68,14 @@ impl PhysicalCandidate { && (self.physical_properties.timing() == ExecutionTiming::IngestionTime || logical.memo.data_arrival() == DataArrival::AtRest || group_properties.scanned_sources.is_empty()); - if self.chain_tumbling_window_width_ms(candidates).is_none() && final_at_every_time + if logical + .memo + .candidate(self.logical_candidate_id) + .expression + .operator + .chain_tumbling_window_width_ms() + .is_none() + && final_at_every_time { return true; } @@ -79,60 +85,23 @@ impl PhysicalCandidate { (window.width_ms, window.phase_ms) } ExecutionContext::Materialized(v) => { - match candidates.physical_candidates[v.0].own_schedule(candidates) { - Some(window) => (window.width_ms, window.phase_ms), + let materialized = candidates.physical_candidates[v.0].logical_candidate_id; + match logical.own_schedule(materialized) { + Some(grid) => (grid.period_ms, grid.phase_ms), None => return false, } } }; - self.own_schedule(candidates).is_some_and(|stored| { - stored.width_ms > 0 - && period_ms.is_multiple_of(stored.width_ms) - && phase_ms % stored.width_ms == stored.phase_ms % stored.width_ms - }) + logical + .own_schedule(self.logical_candidate_id) + .is_some_and(|stored| { + stored.period_ms > 0 + && period_ms.is_multiple_of(stored.period_ms) + && phase_ms % stored.period_ms == stored.phase_ms % stored.period_ms + }) } } } - - /// The times it computes at when materialized across evaluations, on - /// its own schedule: every window end of a chain (its width at phase - /// 0), else its group's evaluation grid (`evaluation_grid_of_group`); - /// `None` when neither is known. - pub fn own_schedule(&self, candidates: &PhysicalASAPMemoCandidates) -> Option { - match self.chain_tumbling_window_width_ms(candidates) { - Some(width_ms) => Some(RefreshWindow { - width_ms, - phase_ms: 0, - }), - None => candidates.logical_candidates.evaluation_grid_of_group - [self.logical_candidate_id.group.0 as usize] - .map(|grid| RefreshWindow { - width_ms: grid.period_ms, - phase_ms: grid.phase_ms, - }), - } - } - - /// Its tumbling-window width when it is a chain - /// (`SummaryTumblingWindowChain`). - pub fn chain_tumbling_window_width_ms( - &self, - candidates: &PhysicalASAPMemoCandidates, - ) -> Option { - match candidates - .logical_candidates - .memo - .candidate(self.logical_candidate_id) - .expression - .operator - { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(tumbling_window_width_ms), - _ => None, - } - } } /// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs index 31f95c13f..c95744fa7 100644 --- a/crates/types/src/stages/physical/execution_context.rs +++ b/crates/types/src/stages/physical/execution_context.rs @@ -4,7 +4,11 @@ //! the physical candidates. use super::candidate::PhysicalCandidateId; +use crate::ir::operator::AtModifier; use crate::plan::materialized_view::RefreshWindow; +use crate::stages::EvaluationGrid; +use crate::workload::QueryWorkloadEntry; +use std::collections::{BTreeSet, HashMap, HashSet}; /// Index of an [`ExecutionContext`] in the workload's list of distinct /// contexts. @@ -26,9 +30,128 @@ pub enum ExecutionContext { Materialized(PhysicalCandidateId), } -/// The evaluation times `phase_ms + k · period_ms`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct EvaluationGrid { - pub period_ms: u64, - pub phase_ms: u64, +impl ExecutionContext { + /// The one context of every ingestion-time execution that is not + /// materialized: each second of arriving rows, which it computes once + /// for every ingestion-time view of every refresh window that reads it. + pub const EVERY_ARRIVING_ROW: Self = Self::IngestionPipeline(RefreshWindow { + width_ms: 1_000, + phase_ms: 0, + }); +} + +/// Whether a `TimeShift` anchored with `at`, read by `queries` (those on +/// one evaluation grid that reach it), is anchored at their evaluation +/// times, so its input runs in their evaluation context, shared with +/// their other reads: `@ end()` with an instant query among them (an +/// instant query's request ends at its evaluation; the input computed at +/// every evaluation of the grid serves the range queries' request ends +/// too). Otherwise it runs once per request, in a context of its own. +/// +/// `@ start()` of an instant query is its evaluation time too; it keeps +/// its request context here, as the user decided for `@ end()` only. +/// +/// Example: `sum_over_time(m[5m] @ end())` and `sum_over_time(m[5m])`, +/// both instant every 60 s: true, so `m[5m]` is computed once per minute +/// for both; the same `@ end()` in a range query alone: false. +pub fn anchor_is_evaluation_time<'e>( + at: AtModifier, + queries: impl IntoIterator, +) -> bool { + at == AtModifier::End + && queries.into_iter().any(|entry| { + entry + .range_evaluation + .is_none_or(|range| range.evaluations_per_request() <= 1) + }) +} + +/// The queries of the execution context on evaluation grid `grid_ms` +/// among `reaching` (the queries reaching a node, by index in `entries`): +/// those evaluated on that grid +/// ([`QueryWorkloadEntry::evaluation_grid_ms`]; 1 ms for a query without +/// an entry). Stage 3 (over memo groups) and `asap_cost`'s `computations` +/// (over plan nodes) pick a context's queries with it. +pub fn queries_of_context( + grid_ms: u64, + reaching: impl IntoIterator, + entries: &[QueryWorkloadEntry], +) -> Vec { + reaching + .into_iter() + .filter(|&query| { + entries + .get(query) + .map_or(1, QueryWorkloadEntry::evaluation_grid_ms) + == grid_ms + }) + .collect() +} + +/// How each query reads the nodes below it, in a plan or a memo: per node, +/// each (query, anchor) whose walk from the query's `starts` down +/// `inputs_of` reaches it, once, skipping (and stopping at) a node +/// `stops_at` keeps. Below a `TimeShift` anchored with `@ end()` or +/// `@ start()` (`anchor_of`, `Operator::request_anchor`) a query reads per request, with that anchor, +/// unless the anchor is its evaluation time ([`anchor_is_evaluation_time`] +/// over the queries on its grid that reach the shift through any node); +/// elsewhere with none. Stage 3 walks memo groups with it and +/// `asap_cost`'s `consumers` plan nodes, so both rate a context by the queries whose +/// walk reaches it. Stage 3 walks every candidate of a group, so a query +/// reading a group through one candidate counts for the group's other +/// candidates too. +/// +/// Example: an instant `sum_over_time(m[5m] @ end())` and a range +/// `sum_over_time(m[5m] @ end() offset 10m)`, both every 60 s: `m[5m]` is +/// read by the instant query with no anchor and by the range query with +/// `@ end()` (its requests' ends are not its evaluations). +pub fn query_reads( + starts: &[Vec], + inputs_of: impl Fn(N) -> Vec, + anchor_of: impl Fn(N) -> Option, + stops_at: impl Fn(N) -> bool, + entries: &[QueryWorkloadEntry], +) -> HashMap)>> { + // The queries reaching each anchored shift, through any node. + let mut reaching_shift: HashMap> = HashMap::new(); + for (query, start) in starts.iter().enumerate() { + let mut seen = HashSet::new(); + let mut stack = start.clone(); + while let Some(id) = stack.pop() { + if seen.insert(id) { + if anchor_of(id).is_some() { + reaching_shift.entry(id).or_default().insert(query); + } + stack.extend(inputs_of(id)); + } + } + } + let in_evaluation_context = |shift: N, query: usize, at: AtModifier| { + let Some(entry) = entries.get(query) else { + return false; + }; + let on_grid = queries_of_context( + entry.evaluation_grid_ms(), + reaching_shift.get(&shift).into_iter().flatten().copied(), + entries, + ); + anchor_is_evaluation_time(at, on_grid.iter().map(|&q| &entries[q])) + }; + let mut reads: HashMap)>> = HashMap::new(); + for (query, start) in starts.iter().enumerate() { + let mut seen = HashSet::new(); + let mut stack: Vec<(N, Option)> = start.iter().map(|&id| (id, None)).collect(); + while let Some((id, anchor)) = stack.pop() { + if stops_at(id) || !seen.insert((id, anchor)) { + continue; + } + reads.entry(id).or_default().push((query, anchor)); + let below = match (anchor, anchor_of(id)) { + (None, Some(at)) if !in_evaluation_context(id, query, at) => Some(at), + (anchor, _) => anchor, + }; + stack.extend(inputs_of(id).into_iter().map(|input| (input, below))); + } + } + reads } diff --git a/crates/types/src/stages/physical/memo_candidates/mod.rs b/crates/types/src/stages/physical/memo_candidates/mod.rs index e799cc98f..1a8c4ce00 100644 --- a/crates/types/src/stages/physical/memo_candidates/mod.rs +++ b/crates/types/src/stages/physical/memo_candidates/mod.rs @@ -6,9 +6,9 @@ use std::collections::HashMap; -use crate::ir::QueryRoot; +use crate::ir::{Operator, QueryRoot}; use crate::stages::{ - LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, + LogicalASAPMemoCandidates, MemoGroupId, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PrunedCandidate, }; @@ -78,16 +78,20 @@ impl PhysicalASAPMemoCandidates { &self.edges[id.0] } - /// The number of inputs of candidate `v`: its logical candidate's - /// operator's children (e.g. 2 for a binary operator). - pub fn input_count(&self, v: PhysicalCandidateId) -> usize { - self.logical_candidates + /// The operator of candidate `v`'s logical candidate. + pub fn operator(&self, v: PhysicalCandidateId) -> &Operator { + &self + .logical_candidates .memo .candidate(self.physical_candidates[v.0].logical_candidate_id) .expression .operator - .children() - .len() + } + + /// The number of inputs of candidate `v`: its logical candidate's + /// operator's children (e.g. 2 for a binary operator). + pub fn input_count(&self, v: PhysicalCandidateId) -> usize { + self.operator(v).children().len() } /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index ffd54b47f..5ec7c5245 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -23,7 +23,7 @@ use crate::ir::{NonASAPOp, TimeRangeKind}; use crate::stages::StorageMedium; -use crate::stages::physical::execution_context::EvaluationGrid; +use crate::stages::EvaluationGrid; use crate::stages::{ CandidateForMemoGroup, LogicalASAPMemoCandidates, Memo, MemoGroupId, diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index 2b3728d4d..c8d45dd71 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -18,6 +18,9 @@ pub use candidate::{ producer_satisfies_physical_requirements, PhysicalCandidate, PhysicalCandidateId, }; pub use edge::{PhysicalEdge, PhysicalEdgeId}; -pub use execution_context::{EvaluationGrid, ExecutionContext, ExecutionContextId}; +pub use execution_context::{ + anchor_is_evaluation_time, queries_of_context, query_reads, ExecutionContext, + ExecutionContextId, +}; pub use memo_candidates::{output_references, PhysicalASAPMemoCandidates}; pub use properties::{Lifetime, PhysicalProperties, PhysicalPropertiesError, StorageMedium}; diff --git a/crates/types/src/stages/physical/properties/mod.rs b/crates/types/src/stages/physical/properties/mod.rs index f59e9b4f9..530df3ace 100644 --- a/crates/types/src/stages/physical/properties/mod.rs +++ b/crates/types/src/stages/physical/properties/mod.rs @@ -92,6 +92,15 @@ impl PhysicalProperties { pub fn is_across_evaluations(&self) -> bool { matches!(self.lifetime, Lifetime::AcrossEvaluations(_)) } + + /// Whether it has at most one consumer per execution context: a + /// query-time result that is not materialized ($\ell$ = None). + /// Consumers in one context that share a result read a `OneEvaluation` + /// or a materialized producer; the consumers of one ingestion run share + /// its rows. + pub fn has_one_reader_per_context(&self) -> bool { + self.lifetime == Lifetime::None && self.timing == ExecutionTiming::QueryTime + } } #[cfg(test)] diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index d876e9ce8..ebc3ea718 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -1,6 +1,7 @@ //! The query workload: each query, its requirements and its recurrence. use crate::accuracy::AccuracyTarget; +use crate::ir::operator::AtModifier; use serde::{Deserialize, Serialize}; use super::data::validate_rate; @@ -415,6 +416,32 @@ pub struct QueryWorkloadEntry { pub range_evaluation: Option, } +impl QueryWorkloadEntry { + /// The grid, in ms, every evaluation of the query falls on + /// ([`QueryRecurrence::evaluation_grid_ms`] of its range evaluation), + /// the grid of its evaluation context; 1 ms when not known, a grid on + /// which only producers stored at every time deliver. + pub fn evaluation_grid_ms(&self) -> u64 { + self.recurrence + .evaluation_grid_ms(self.range_evaluation) + .unwrap_or(1) + } + + /// The grid, in ms, of the times its requests anchor `@ end()` or + /// `@ start()` (`at`) at: its request end times, shifted back by its + /// range-evaluation span for `@ start()`; 1 ms when not known. + /// + /// Example: a query every 60 s over the last hour: `@ end()` 60,000 + /// ms, `@ start()` gcd(60,000, 3,600,000) = 60,000 ms. + pub fn request_time_grid_ms(&self, at: AtModifier) -> u64 { + let end_ms = self.recurrence.evaluation_grid_ms(None).unwrap_or(1); + match (at, self.range_evaluation) { + (AtModifier::Start, Some(range)) => greatest_common_divisor(end_ms, range.span_ms), + _ => end_ms, + } + } +} + impl From<&BatchEntry> for QueryWorkloadEntry { fn from(entry: &BatchEntry) -> Self { Self { From 5f3141ab76efb507e9c1dd67a5e2ed936c20f7fd Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:06:11 +0000 Subject: [PATCH 17/23] test(cost,integration): an ingestion-time reader of raw rows kept at ingestion reads one batch; two queries reading one anchored group (#1207 PR 7) --- .../cost/src/plan/execution_demand/tests.rs | 64 +++++++++++++++++++ .../tests/stage3_objective.rs | 27 ++++++++ 2 files changed, 91 insertions(+) diff --git a/crates/cost/src/plan/execution_demand/tests.rs b/crates/cost/src/plan/execution_demand/tests.rs index 9fba374d6..832d15d11 100644 --- a/crates/cost/src/plan/execution_demand/tests.rs +++ b/crates/cost/src/plan/execution_demand/tests.rs @@ -444,3 +444,67 @@ fn an_instant_at_end_input_runs_in_the_evaluation_context() { close(rates[&id].1, 1.0 / 60.0); } } + +/// An ingestion-time chain of 60 s windows over `m` (1,000 rows per +/// second), read back 5 minutes at query time every 60 s, reading either +/// a `Scan` that is not materialized or raw rows `m` kept at ingestion for +/// an hour: either way it folds in one arriving batch per second +/// (`reads_one_batch_of`), so `plan_cost` charges it the same, not as if +/// it read the hour of stored rows. +#[test] +fn an_ingestion_time_reader_of_raw_rows_kept_at_ingestion_reads_one_batch() { + use asap_types::ir::operator::Reduction; + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::FieldDataType; + use asap_types::stages::StorageMedium; + use asap_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; + use asap_types::workload::{DataWorkload, DurationMs}; + let ingestion = ExecutionTiming::IngestionTime; + let chain = Operator::ASAP(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { + child: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }); + let mut data = DataWorkload::sampled_every(["m".to_string()], DurationMs(1_000)); + for facts in data.sources.values_mut() { + facts.series_count = Some(1_000); + } + let chain_cost = |scan_lifetime: Lifetime, scan_retention_ms: Option| { + let mut stored_chain = node( + 1, + chain.clone(), + ingestion, + Lifetime::AcrossEvaluations(StorageMedium::Memory), + ); + stored_chain.retention_ms = Some(300_000); + let mut raw = node(0, scan(), ingestion, scan_lifetime); + raw.retention_ms = scan_retention_ms; + let dag = PhysicalASAPDAG::new( + vec![ + raw, + stored_chain, + node(2, last(300, 1), ExecutionTiming::QueryTime, Lifetime::None), + ], + vec![2], + vec![], + ) + .unwrap(); + let (cost, _) = crate::plan_cost( + &dag, + &entries(&[fixed(60_000, 0)], None), + &data, + &crate::CostInputs::default(), + ) + .unwrap(); + cost.per_node[&1].cost_per_second.unwrap() + }; + let over_arriving_rows = chain_cost(Lifetime::None, None); + let over_kept_rows = chain_cost( + Lifetime::AcrossEvaluations(StorageMedium::Memory), + Some(3_600_000), + ); + close(over_kept_rows, over_arriving_rows); +} diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index 34934bc6e..1e48dd939 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -577,3 +577,30 @@ fn the_objective_is_the_plan_cost_on_an_instant_and_a_shifted_range_at_end() { selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1, ); } + +/// Two range queries every 300 s on one grid reach one `@ end()` group, +/// `m[5m] @ end()`, and may read it through different candidates: a sum +/// over 10 minutes at 30 s steps and a median over an hour at 60 s steps, +/// accuracy guarantees ignored (so the median may read a sketch). Stage 3 +/// rates the anchored context from the memo, `plan_cost` from the plan's +/// walk; the optimum is the plan's cost. +#[test] +fn the_objective_is_the_plan_cost_when_two_queries_read_one_anchored_group() { + let w = over_m(vec![ + requested( + "sum_over_time(m[5m] @ end())", + 300_000, + Some((30_000, 600_000)), + ), + requested( + "quantile_over_time(0.5, m[5m] @ end())", + 300_000, + Some((60_000, 3_600_000)), + ), + ]); + let models = PlanningModels::builtin().with_ignore_accuracy_guarantees(true); + assert_equal( + "two queries reading one anchored group", + selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1, + ); +} From bb62f8116d38500ae31a4b27730c56b7f4b2afb4 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:46:00 +0000 Subject: [PATCH 18/23] perf(stage3): a copy of an equivalent evaluation context shares the original's variables (#1207 PR 7) The same queries on several grids (one user every 60 s, another every 300 s) give one evaluation context per grid with the same executions and edges, each cost a fixed multiple of the other's. The MILP chose each copy freely, so HiGHS branched over both: c16 130 s, RangeHeavy-8users 222 s on seed 0. copies_in_equivalent_contexts finds such copies (same candidates, kept alike, same memo edges from corresponding producers, paired queries with the same outputs, exact-only groups and span, costs proportional), and selection_variables gives a copy's executions, edges and queries the original's x, y and z. Exact: some optimum selects alike in both copies (give both the cheaper one's selection; retention and raw data do not grow). Plan costs are unchanged; c16 drops to about 10 s. --- .../src/milp/dominance.rs | 212 +++++++++++++++++- .../src/milp/model/mod.rs | 49 ++-- .../src/milp/model/tests.rs | 92 ++++++++ 3 files changed, 333 insertions(+), 20 deletions(-) diff --git a/crates/stage3-plan-selection/src/milp/dominance.rs b/crates/stage3-plan-selection/src/milp/dominance.rs index d5566c008..3d5d46257 100644 --- a/crates/stage3-plan-selection/src/milp/dominance.rs +++ b/crates/stage3-plan-selection/src/milp/dominance.rs @@ -1,11 +1,14 @@ -//! Executions left out of the selection MILP because another execution -//! dominates them: one with $\ell$ = None whose `OneEvaluation` twin does -//! all it does at no more cost. +//! Executions the selection MILP need not choose freely because another +//! does as well: one with $\ell$ = None whose `OneEvaluation` twin does +//! all it does at no more cost is left out +//! ([`dominated_by_one_evaluation_twin`]); a copy of an equivalent +//! evaluation context shares the original's variables +//! ([`copies_in_equivalent_contexts`]). -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::Lifetime; +use asap_types::stages::{ExecutionContext, ExecutionContextId, Lifetime, PhysicalEdgeId}; use super::model::SelectionInputs; use crate::selectable::QueryId; @@ -103,3 +106,202 @@ pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> } dominated } + +/// Per execution, edge and query, the one whose variables it shares +/// (itself when none): a root evaluation context (no request anchor) +/// equivalent to an earlier one is a copy of it, and so are its +/// executions, edges and queries. Two such contexts are equivalent when +/// their executions are of the same candidates, kept alike, their edges +/// the same memo edges from producers that correspond (in the context, or +/// the same execution outside every evaluation context), their queries +/// pair up with the same output executions, exact-only groups and +/// range-evaluation span, and every cost in one is the same multiple +/// $\alpha$ of the other's (to a relative 1e-9, the rounding of rate times +/// cost per run). +/// +/// Exact: from an optimal plan, give both contexts the selection of the +/// one whose own cost $A$ or $B$ is lower, say $A$. The copied context +/// then costs $\alpha A \le \alpha B$, the retention $h_v$ and raw data +/// $r_s$ (each the most of their readers' needs, equal in both copies) do +/// not grow, and an execution outside them that only the other selection +/// read is left out at no cost; with $B \le A$ symmetrically. So some +/// optimum selects alike in both, and sharing their variables keeps it. +/// HiGHS no longer branches over each context's copy. +/// +/// Example: the same queries every 60 s and every 300 s (one user each): +/// the 300 s context's executions cost 1/5 of the 60 s ones' and share +/// their variables. +pub(crate) fn copies_in_equivalent_contexts( + inputs: &SelectionInputs<'_>, + kept: &[bool], +) -> (Vec, Vec, Vec) { + let selectable = inputs.selectable; + let costs = inputs.costs; + let executions = selectable.executions(); + let edges = selectable.edges(); + let mut original_of_execution: Vec = (0..executions.len()).collect(); + let mut original_of_edge: Vec = (0..edges.len()).collect(); + let mut original_of_query: Vec = (0..inputs.entries.len()).map(QueryId).collect(); + let is_root_evaluation = |context: ExecutionContextId| { + matches!( + selectable.context(context), + ExecutionContext::Evaluation { .. } + ) && selectable.request_anchor(context).is_none() + }; + let is_shared = |context: ExecutionContextId| { + !matches!( + selectable.context(context), + ExecutionContext::Evaluation { .. } + ) + }; + // Each root evaluation context's executions (ascending, so by + // candidate) and queries. + let mut executions_of: BTreeMap> = BTreeMap::new(); + for (index, execution) in executions.iter().enumerate() { + if is_root_evaluation(execution.context) { + executions_of + .entry(execution.context) + .or_default() + .push(index); + } + } + let mut outputs_of: BTreeMap> = BTreeMap::new(); + for (query, answering) in selectable.answering() { + outputs_of.entry(*query).or_default().push(answering); + } + let mut queries_of: BTreeMap> = BTreeMap::new(); + for (&query, outputs) in &outputs_of { + // A query's outputs are answered in its own context. + if let Some(&index) = outputs.first().and_then(|answering| answering.first()) { + queries_of + .entry(executions[index].context) + .or_default() + .push(query); + } + } + let edge_at: HashMap<(usize, PhysicalEdgeId, usize), usize> = edges + .iter() + .enumerate() + .map(|(position, edge)| ((edge.consumer, edge.edge, edge.producer), position)) + .collect(); + // Whether `b` is `a`'s cost times `alpha` (fixed by the first pair + // with a cost). + let proportional = |alpha: &mut Option, a: f64, b: f64| match (a == 0.0, b == 0.0) { + (true, true) => true, + (false, false) => { + let ratio = *alpha.get_or_insert(b / a); + (b - ratio * a).abs() <= 1e-9 * b.abs() + } + _ => false, + }; + let contexts: Vec = executions_of.keys().copied().collect(); + let mut originals: Vec = Vec::new(); + for © in &contexts { + let copied = originals.iter().find_map(|&original| { + let (mine, theirs) = (&executions_of[©], &executions_of[&original]); + if mine.len() != theirs.len() { + return None; + } + let mut alpha = None; + let mut mapped: HashMap = HashMap::new(); + for (&b, &a) in mine.iter().zip(theirs) { + let same = executions[a].candidate == executions[b].candidate + && kept[a] == kept[b] + && costs.least_need_ms_per_execution[a] == costs.least_need_ms_per_execution[b] + && selectable.input_count(a) == selectable.input_count(b) + && selectable.edges_out_of(a).len() == selectable.edges_out_of(b).len() + && match (&costs.per_execution[a], &costs.per_execution[b]) { + (Ok(cost_a), Ok(cost_b)) => proportional(&mut alpha, *cost_a, *cost_b), + (Err(_), Err(_)) => true, + _ => false, + }; + if !same { + return None; + } + mapped.insert(b, a); + } + let mut edges_mapped: Vec<(usize, usize)> = Vec::new(); + for (&b, &a) in mine.iter().zip(theirs) { + for input in 0..selectable.input_count(b) { + let (into_b, into_a) = ( + selectable.edges_into(b, input), + selectable.edges_into(a, input), + ); + if into_b.len() != into_a.len() { + return None; + } + for &position in into_b { + let edge = edges[position]; + let producer = match mapped.get(&edge.producer) { + Some(&producer) => producer, + None if is_shared(executions[edge.producer].context) => edge.producer, + None => return None, + }; + let twin = *edge_at.get(&(a, edge.edge, producer))?; + let same = match (costs.per_edge[twin], costs.per_edge[position]) { + (Some(cost_a), Some(cost_b)) => { + proportional(&mut alpha, cost_a, cost_b) + } + (None, None) => true, + _ => false, + }; + if !same { + return None; + } + edges_mapped.push((position, twin)); + } + } + } + // Each query of `copy` pairs with one of `original` reading the + // same output executions, exact-only groups and span. + let signature = |query: QueryId, map: &dyn Fn(usize) -> usize| { + let outputs: Vec> = outputs_of[&query] + .iter() + .map(|answering| answering.iter().map(|&index| map(index)).collect()) + .collect(); + let entry = &inputs.entries[query.0]; + ( + outputs, + inputs + .physical + .logical_candidates + .exact_only_groups_of_query + .get(query.0), + entry.range_evaluation.map(|range| range.span_ms), + ) + }; + let (my_queries, their_queries) = ( + queries_of.get(©).map_or(&[][..], Vec::as_slice), + queries_of.get(&original).map_or(&[][..], Vec::as_slice), + ); + if my_queries.len() != their_queries.len() { + return None; + } + let mut unpaired: Vec = their_queries.to_vec(); + let mut queries_mapped = Vec::new(); + for &query in my_queries { + let mine = signature(query, &|index| mapped[&index]); + let at = unpaired + .iter() + .position(|&theirs| signature(theirs, &|index| index) == mine)?; + queries_mapped.push((query, unpaired.swap_remove(at))); + } + Some((original, mapped, edges_mapped, queries_mapped)) + }); + match copied { + Some((_, mapped, edges_mapped, queries_mapped)) => { + for (b, a) in mapped { + original_of_execution[b] = a; + } + for (b, a) in edges_mapped { + original_of_edge[b] = a; + } + for (b, a) in queries_mapped { + original_of_query[b.0] = a; + } + } + None => originals.push(copy), + } + } + (original_of_execution, original_of_edge, original_of_query) +} diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 33a52f12c..99df0de6c 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -28,7 +28,7 @@ use asap_types::ir::operator::Source; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; -use super::dominance::dominated_by_one_evaluation_twin; +use super::dominance::{copies_in_equivalent_contexts, dominated_by_one_evaluation_twin}; use super::exact_only::{exact_only_constraints, exact_only_variables}; use super::graph::Components; use super::retention::{least_retention_ms, retention_constraints, retention_variables}; @@ -79,7 +79,10 @@ pub(crate) struct SelectionVariables { /// $x$, $y$, $z$, $t$, $h$ and $r$ over `inputs`, added to `problem`: $x$ /// fixed at 0 for an execution that is not `kept` (not costed, or dearer -/// than a feasible plan), and no $y$ on an edge of one. +/// than a feasible plan), and no $y$ on an edge of one. The executions, +/// edges and queries of a copy of an equivalent evaluation context take the +/// original's $x$, $y$ and $z$ ([`copies_in_equivalent_contexts`]), so +/// their costs add up on one variable. fn selection_variables( inputs: &SelectionInputs<'_>, kept: &[bool], @@ -87,26 +90,40 @@ fn selection_variables( problem: &mut ProblemVariables, ) -> SelectionVariables { let selectable = inputs.selectable; - let x = kept - .iter() - .map(|&kept| match kept { - true => problem.add(variable().binary()), - false => problem.add(variable().min(0).max(0)), - }) - .collect(); - let y = selectable + let (original_of_execution, original_of_edge, original_of_query) = + copies_in_equivalent_contexts(inputs, kept); + let mut x: Vec = Vec::with_capacity(kept.len()); + for (index, &kept) in kept.iter().enumerate() { + let variable = match (original_of_execution[index], kept) { + (original, _) if original != index => x[original], + (_, true) => problem.add(variable().binary()), + (_, false) => problem.add(variable().min(0).max(0)), + }; + x.push(variable); + } + let mut y: BTreeMap = BTreeMap::new(); + for (position, (edge, cost)) in selectable .edges() .iter() .zip(&inputs.costs.per_edge) .enumerate() - .filter(|(_, (edge, cost))| cost.is_some() && kept[edge.consumer] && kept[edge.producer]) - .map(|(position, _)| (position, problem.add(variable().binary()))) - .collect(); + { + if cost.is_some() && kept[edge.consumer] && kept[edge.producer] { + let variable = match original_of_edge[position] { + original if original != position => y[&original], + _ => problem.add(variable().binary()), + }; + y.insert(position, variable); + } + } let mut z = BTreeMap::new(); for (query, executions) in selectable.answering() { for &execution in executions { - z.entry((*query, execution)) + let original = (original_of_query[query.0], original_of_execution[execution]); + let variable = *z + .entry(original) .or_insert_with(|| problem.add(variable().binary())); + z.insert((*query, execution), variable); } } // t only inside a strongly connected component of the selectable @@ -356,7 +373,9 @@ pub(crate) enum SelectionMilpError { /// [`raw_data_disk_rows`], (R) and (X), with HiGHS at gap 0 on one /// thread. The executions dearer than a feasible plan /// ([`dearer_than_a_feasible_plan`]) or dominated by their `OneEvaluation` -/// twin ([`dominated_by_one_evaluation_twin`]) are left out first, with +/// twin ([`dominated_by_one_evaluation_twin`]) are left out first (and a +/// copy of an equivalent context shares its original's variables, +/// [`copies_in_equivalent_contexts`]), with /// the $h_v$ of a candidate whose /// executions are all left out, then the coefficients kept are scaled so /// the largest is 1,000 (HiGHS's tolerances are absolute). diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 68607ac14..4fb63ae4b 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -771,3 +771,95 @@ fn a_none_execution_its_one_evaluation_twin_dominates_is_left_out() { assert_eq!(dominated(&[1.0, 2.0, 1.0], true), (vec![], vec![0, 2])); assert_eq!(dominated(&[1.0, 1.0, 1.0], false), (vec![], vec![0, 2])); } + +/// The same query `m[5m]` every 10 s and every 20 s: two evaluation +/// contexts with the same executions and edges, each cost in the 20 s one +/// half the 10 s one's (half the runs). The 20 s context is a copy of the +/// 10 s one ([`copies_in_equivalent_contexts`]): its executions, edges and +/// query share their variables, and the optimum reads the scan's +/// `OneEvaluation` candidate in both (1.5 + 0 against 1 + 2), costing +/// 2.5 + 1.25. With one cost of the 20 s context not half, they are not +/// copies. +#[test] +fn a_context_equivalent_to_another_shares_its_variables() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = + selection_inputs(&physical, &entries, &[1.0, 1.5, 1.0], &[((0, 2), 2.0)], &[]); + let executions = selectable.executions(); + let every_20s = |index: usize| { + matches!( + selectable.context(executions[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 20_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_20s(index) { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + for (position, edge) in selectable.edges().iter().enumerate() { + if every_20s(edge.consumer) { + costs.per_edge[position] = costs.per_edge[position].map(|cost| cost / 2.0); + } + } + let kept = vec![true; executions.len()]; + let copies = |costs: &ExecutionCosts| { + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs, + }; + copies_in_equivalent_contexts(&inputs, &kept) + }; + let (original_of_execution, original_of_edge, original_of_query) = copies(&costs); + for (index, &original) in original_of_execution.iter().enumerate() { + match every_20s(index) { + true => { + assert!(!every_20s(original)); + assert_eq!(executions[original].candidate, executions[index].candidate); + } + false => assert_eq!(original, index), + } + } + for (position, &original) in original_of_edge.iter().enumerate() { + let edge = selectable.edges()[position]; + assert_eq!(original != position, every_20s(edge.consumer)); + assert_eq!(selectable.edges()[original].edge, edge.edge); + } + assert_eq!(original_of_query, vec![QueryId(0), QueryId(0)]); + + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + assert_eq!(candidates(&solved), vec![1, 1, 2, 2]); + assert!((solved.cost_per_second_without_request_overhead - 3.75).abs() < 1e-9); + assert_eq!(solved.variable_count, 3 + 2 + 1); + + let first_of_20s = (0..executions.len()) + .find(|&index| every_20s(index)) + .unwrap(); + costs.per_execution[first_of_20s] = Ok(10.0); + let (original_of_execution, _, _) = copies(&costs); + assert!(original_of_execution + .iter() + .enumerate() + .all(|(index, &original)| original == index)); +} From a1abc86159ebd8582d522dddc12764d2463b7b58 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:51:35 +0000 Subject: [PATCH 19/23] refactor(types,cost,stage3): one refresh rule for an ingestion-time stored result (#1207 PR 7) Three rules said when an ingestion-time stored result refreshes and disagreed: Stage 3 (own_schedule: a chain at phase 0, anything else its group's evaluation grid), the installed plan (refresh_window: a chain (w, s mod w), anything else the union of the windows below it) and the cost model (maintained_window_ms: the first range or chain below). An ingestion-time sum(TimeRange(5m)) read every 60 s ran its inputs on a 60 s pipeline and served the 60 s reader, while the plan refreshed it every 5 min; a chain over TimeShift(30s) had phase 0 in Stage 3 and 30 s in the plan. The plan's rule is the one rule: RefreshWindow::computed_from over any TimeIndexedPlan. PhysicalASAPDAG::refresh_window, Stage 3's ingestion pipeline context, the stored times delivers_on checks (PhysicalASAPMemoCandidates::stored_times; delivers_on moves to the candidates, whose data it reads) and both cost paths (plan_cost and Stage 3's ingestion_window_ms, its width) use it. The candidates implement TimeIndexedPlan; Stage 3's per-edge Candidates delegates to it. shift_back_ms_below follows every alternative producer and needs them to agree. maintained_window_ms and refresh_windows are deleted. --- crates/cost/src/lib.rs | 6 +- crates/cost/src/node/usage/mod.rs | 46 +----- crates/cost/src/node/usage/tests.rs | 10 +- crates/cost/src/plan/cost.rs | 12 +- .../stage2-physical-optimization/src/group.rs | 2 +- .../stage2-physical-optimization/src/lib.rs | 2 +- .../src/tests/candidates.rs | 2 +- crates/stage3-plan-selection/src/cost.rs | 37 ++--- crates/stage3-plan-selection/src/retention.rs | 34 +---- .../stage3-plan-selection/src/selectable.rs | 21 +-- .../types/src/plan/materialized_view/mod.rs | 22 ++- .../types/src/plan/materialized_view/views.rs | 33 +--- crates/types/src/plan/time_window.rs | 53 ++++--- crates/types/src/stages/logical/group/mod.rs | 4 +- crates/types/src/stages/physical/candidate.rs | 90 +---------- .../src/stages/physical/execution_context.rs | 2 +- .../stages/physical/memo_candidates/mod.rs | 141 +++++++++++++++++- .../stages/physical/memo_candidates/tests.rs | 138 ++++++++++++++++- crates/types/src/workload/query.rs | 2 +- crates/types/tests/memo_group_properties.rs | 2 +- .../asap-aware-mapping-contracts.md | 2 +- 21 files changed, 380 insertions(+), 281 deletions(-) diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index e170a6940..4ea46e909 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -20,7 +20,7 @@ //! Stage 3 (`asap-stage3-plan-selection`) costs each candidate as one node //! with the same function and the rules `plan_cost` applies: //! [`plan::execution_demand::executions_per_second`], -//! [`maintained_window_ms`], [`read_back_ms`], +//! `RefreshWindow::computed_from` (an ingestion-time node's window), [`read_back_ms`], //! [`raw_data_disk_cost_per_second`] and //! [`request_overhead_cost_per_second`]. mod measured; @@ -40,8 +40,8 @@ pub use model::{ }; pub use node::operator_statistics::EdgeStatistics; pub use node::usage::{ - maintained_window_ms, node_resource_usage_and_cost, output_interval_ms, reads_one_batch_of, - summary_type_name, NodeCost, NodeResourceUsageAndCost, ProducerOutput, + node_resource_usage_and_cost, output_interval_ms, reads_one_batch_of, summary_type_name, + NodeCost, NodeResourceUsageAndCost, ProducerOutput, }; pub use plan::cost::{ plan_cost, raw_data_disk_cost_per_second, request_overhead_cost_per_second, PlanCost, diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 146c73227..a4af944c0 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -223,47 +223,6 @@ pub fn reads_one_batch_of( /// refreshed once per second. const ONE_SECOND_MS: u64 = 1_000; -/// The window an ingestion-time grouping `node` maintains, in ms: a -/// tumbling-window chain's tumbling window, a `TimeRange`'s range, or the -/// first `TimeRange` or chain below it through filters, projections and -/// time shifts (a view over an ingestion-time chain is refreshed once per -/// chain window; `input_of` gives a node's first input, `operator_of` its -/// operator). `None` when there is none. Over plan nodes ([`plan_cost`]) -/// or memo groups (Stage 3, before any plan exists). -/// -/// Example: `Aggregate(Sum) <- Filter <- TimeRange(1m) <- Scan`: 60,000; -/// `TimeShift(1h) <- SummaryTumblingWindowChain(1m)`: 60,000. -/// -/// [`plan_cost`]: crate::plan_cost -pub fn maintained_window_ms<'o, N: Copy + Ord, C: 'o>( - node: N, - operator_of: impl Fn(N) -> &'o Operator, - input_of: impl Fn(N) -> Option, -) -> Option { - let range_ms = |range: &std::time::Duration| u64::try_from(range.as_millis()).ok(); - if let Some(width_ms) = operator_of(node).chain_tumbling_window_width_ms() { - return Some(width_ms); - } - if let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = operator_of(node) { - return range_ms(range); - } - let mut seen = BTreeSet::new(); - let mut below = input_of(node)?; - while seen.insert(below) { - if let Some(width_ms) = operator_of(below).chain_tumbling_window_width_ms() { - return Some(width_ms); - } - match operator_of(below) { - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => return range_ms(range), - Operator::NonASAP( - NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, - ) => below = input_of(below)?, - _ => return None, - } - } - None -} - /// The time between two outputs of a node running `payload` that keeps /// rows, which its retention is divided by: a chain's tumbling window, /// else `window_ms` (the window it maintains), else one second of @@ -280,8 +239,9 @@ pub fn output_interval_ms(payload: &Operator, window_ms: Option) -> u /// [`plan_cost`]; Stage 3 calls it per candidate): `producers` are its /// producers in edge order, `rate` its `(multiplier, per_second)` /// ([`computations`], [`executions_per_second`]), and `window_ms` the time -/// window one execution covers: the window an ingestion-time node -/// maintains ([`maintained_window_ms`]), or a query-time `Scan`'s read, the +/// window one execution covers: an ingestion-time node's refresh window +/// width (`RefreshWindow::computed_from`, the rule the installed plan +/// refreshes it on), or a query-time `Scan`'s read, the /// most offset + range its consumers read ([`read_back_ms`] without a span; /// `None`: it reads what it retains). A materialized node keeps what its /// `retention_ms` ($h_v$) covers: a tumbling-window chain its windows in diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index a13acb752..27f781a29 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -415,7 +415,8 @@ fn a_view_without_retention_is_not_costed() { /// A view over an ingestion-time chain of 60 s windows is refreshed once /// per chain window, not once per second: `TimeShift(1h)` over the chain -/// maintains the chain's 60 s window ([`maintained_window_ms`]), and, +/// is refreshed on the chain's 60 s window (its refresh window, +/// `RefreshWindow::computed_from`, is the chain's), and, /// retained for 1 h 5 m (its 5-minute reader and its offset), keeps 65 /// windows' outputs and the one being built, as many as the chain below /// it, not 3,901 seconds' worth. @@ -444,12 +445,7 @@ fn a_view_over_an_ingestion_time_chain_keeps_one_output_per_chain_window() { child: 0, })) }; - let operator_of = |id: u32| match id { - 0 => &chain.payload, - _ => &shift.payload, - }; - let window_ms = maintained_window_ms(1, operator_of, |id| (id == 1).then_some(0)); - assert_eq!(window_ms, Some(60_000)); + let window_ms = Some(60_000); let one_window_of_states = ProducerOutput { payload: &chain.payload, output_schema: &chain.output_schema, diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index dd4049a96..599b19dc8 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -1,7 +1,7 @@ //! Costing a physical plan DAG node by node: each node's resource usage //! ([`node_resource_usage_and_cost`]), then the plan's usage, latency and //! cost ([`plan_cost`]). Stage 3 costs its candidates with the same -//! function and rules (`executions_per_second`, `maintained_window_ms`, +//! function and rules (`executions_per_second`, `RefreshWindow::computed_from`, //! `read_back_ms`, [`raw_data_disk_cost_per_second`], //! [`request_overhead_cost_per_second`]), so its objective is the //! `plan_cost` of the plan it selects. @@ -11,7 +11,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use crate::node::analytical::AnalyticalCostError; use crate::node::operator_statistics::EdgeStatistics; use crate::node::state_bytes::{sketch_shape, StateInputRows}; -use crate::node::usage::{cost_source, maintained_window_ms, reads_one_batch_of}; +use crate::node::usage::{cost_source, reads_one_batch_of}; use crate::plan::execution_demand::{computations, evaluation_rate, query_starts}; use crate::plan::read_back::read_back_ms; use crate::{lookup, StateFamily}; @@ -143,11 +143,9 @@ pub fn plan_cost( }) .collect(); let window_ms = match node.output_state.timing { - ExecutionTiming::IngestionTime => maintained_window_ms( - node.id, - |id| &nodes[&id].payload, - |id| producers_of.get(&id)?.first().copied(), - ), + ExecutionTiming::IngestionTime => { + dag.refresh_window(node.id).map(|window| window.width_ms) + } ExecutionTiming::QueryTime => scan_read_ms(dag, node), }; let node_costed = node_resource_usage_and_cost( diff --git a/crates/stage2-physical-optimization/src/group.rs b/crates/stage2-physical-optimization/src/group.rs index 11ca50a94..b99ef844e 100644 --- a/crates/stage2-physical-optimization/src/group.rs +++ b/crates/stage2-physical-optimization/src/group.rs @@ -64,7 +64,7 @@ pub(super) fn ingestion_time_offered_per_group( /// `AcrossEvaluations`. It does for a chain, for a group evaluated on a /// known grid (`evaluation_grid_of_group`) and for a time-independent /// group (its stored result is read wherever it is final, -/// `PhysicalCandidate::delivers_on`), unless +/// `PhysicalASAPMemoCandidates::delivers_on`), unless /// the group is read under an absolute `@ T` /// ([`read_at_absolute_time_per_group`]). pub(super) fn across_evaluations_offered_per_group( diff --git a/crates/stage2-physical-optimization/src/lib.rs b/crates/stage2-physical-optimization/src/lib.rs index b3cc32315..66e05aa4a 100644 --- a/crates/stage2-physical-optimization/src/lib.rs +++ b/crates/stage2-physical-optimization/src/lib.rs @@ -44,7 +44,7 @@ use candidate::{offered_physical_properties, OfferedPhysicalProperties}; /// outside a dimension's declared value set (`AcrossEvaluations` on a /// candidate of a time-dependent group with no known grid) is not offered /// and not recorded. Costs nothing, and cannot fail; alignment with each -/// execution context is Stage 3's check (`PhysicalCandidate::delivers_on`). +/// execution context is Stage 3's check (`PhysicalASAPMemoCandidates::delivers_on`). pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, data: &DataWorkload, diff --git a/crates/stage2-physical-optimization/src/tests/candidates.rs b/crates/stage2-physical-optimization/src/tests/candidates.rs index e98073b93..02ec2887d 100644 --- a/crates/stage2-physical-optimization/src/tests/candidates.rs +++ b/crates/stage2-physical-optimization/src/tests/candidates.rs @@ -31,7 +31,7 @@ use crate::test_fixtures::{ /// materialized across evaluations. The scan and the chain are /// time-independent, so ingestion time is offered too, and so is /// `AcrossEvaluations` without a grid: kept up to date at ingestion time, -/// their result is final at any time (`PhysicalCandidate::delivers_on`). A +/// their result is final at any time (`PhysicalASAPMemoCandidates::delivers_on`). A /// summary build is not offered (IT, None): fed at ingestion time /// it keeps its state, so the chain is offered (IT, AcrossEvaluations) /// only; the scan's (IT, None) feeds the chain built at ingestion time. diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 19b1963dc..a6f5c61f3 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -55,12 +55,13 @@ use asap_cost::plan::execution_demand::{ context_executions_per_second, executions_per_second as rate_of_materialized, }; use asap_cost::{ - maintained_window_ms, node_resource_usage_and_cost, output_interval_ms, - raw_data_disk_cost_per_second, CostInputs, NodeResourceUsageAndCost, ProducerOutput, + node_resource_usage_and_cost, output_interval_ms, raw_data_disk_cost_per_second, CostInputs, + NodeResourceUsageAndCost, ProducerOutput, }; use asap_types::ir::operator::Source; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::plan::materialized_view::RefreshWindow; use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, }; @@ -644,8 +645,8 @@ fn logical_output<'a: 'b, 'b>( } /// Candidate `v`'s usage as `node` over `inputs`, computed `per_second` -/// times per second, over the window an ingestion-time grouping maintains -/// ([`asap_cost::maintained_window_ms`], over the memo's groups) or, for a +/// times per second, over its refresh window at ingestion time +/// ([`ingestion_window_ms`]) or, for a /// query-time `Scan`, `scan_read_ms` per execution. #[allow(clippy::too_many_arguments)] fn cost_node<'a>( @@ -666,32 +667,16 @@ fn cost_node<'a>( .map_err(|error| error.to_string()) } -/// The window ingestion-time candidate `v` maintains, the time between -/// two of its outputs: a chain its own window, any other grouping the -/// first range or chain below its group ([`asap_cost::maintained_window_ms`], -/// over the memo's groups); `None` without one (one output per second). +/// The time between two outputs of ingestion-time candidate `v`: the +/// width of its refresh window, by the rule the installed plan refreshes +/// it on (`RefreshWindow::computed_from` over the candidates, so +/// `plan_cost` charges the same); `None` without one (one output per +/// second). fn ingestion_window_ms( physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId, ) -> Option { - let memo = &physical.logical_candidates.memo; - let group = physical.physical_candidates[v.0].logical_candidate_id.group; - physical - .operator(v) - .chain_tumbling_window_width_ms() - .or_else(|| { - maintained_window_ms( - group, - |group: MemoGroupId| &memo.groups()[group.0 as usize].expression, - |group| { - memo.groups()[group.0 as usize] - .expression - .children() - .first() - .map(|g| **g) - }, - ) - }) + RefreshWindow::computed_from(physical, v).map(|window| window.width_ms) } /// A candidate's cost per second from its usage under cost model 1: its diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index e4328ed28..829a4cf9d 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -8,7 +8,7 @@ //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 5, "Cost and retention"). -use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::ir::Operator; use asap_types::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; use asap_types::stages::{ MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, @@ -102,11 +102,9 @@ fn read_back( } /// Stage 3's candidates as the time walks see them above `edge`: its -/// producer read by its consumer only, any other candidate by every -/// candidate that may read it (its admissible edges out); each candidate's -/// admissible producers as its inputs, its operator at its timing and -/// lifetime ([`InputTimeAccess::of`]), and a current-time read of a -/// `Scan` (raw data) needing every row. +/// producer read by its consumer only, any other candidate as +/// `PhysicalASAPMemoCandidates` sees it (`TimeIndexedPlan`: by every +/// candidate that may read it, over its admissible edges out). struct Candidates<'a> { physical: &'a PhysicalASAPMemoCandidates, edge: &'a PhysicalEdge, @@ -119,36 +117,20 @@ impl TimeIndexedPlan for Candidates<'_> { fn readers_of(&self, v: PhysicalCandidateId) -> Vec { match v == self.edge.producer_candidate_id { true => vec![self.edge.consumer_candidate_id], - false => self - .physical - .admissible_edges_out_of(v) - .iter() - .map(|&out| self.physical.edge(out).consumer_candidate_id) - .collect(), + false => self.physical.readers_of(v), } } fn inputs_of(&self, v: PhysicalCandidateId) -> Vec { - (0..self.physical.input_count(v)) - .flat_map(|input| self.physical.admissible_edges_into(v, input)) - .map(|&edge| self.physical.edge(edge).producer_candidate_id) - .collect() + self.physical.inputs_of(v) } fn input_time_access(&self, v: PhysicalCandidateId) -> InputTimeAccess { - let properties = self.physical.physical_candidates[v.0].physical_properties; - InputTimeAccess::of( - self.operator_of(v), - properties.timing(), - properties.lifetime(), - ) + self.physical.input_time_access(v) } fn current_time_read_needs_every_row(&self, v: PhysicalCandidateId) -> bool { - matches!( - self.operator_of(v), - Operator::NonASAP(NonASAPOp::Scan { .. }) - ) + self.physical.current_time_read_needs_every_row(v) } fn operator_of(&self, v: PhysicalCandidateId) -> &Operator { diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index 3a3b36c01..37cfd3a2c 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -27,7 +27,7 @@ //! (`asap_types::stages::anchor_is_evaluation_time`). //! - An input of a consumer in $\kappa$ reads a producer, over an admissible //! edge, only where the producer delivers on $\kappa$ -//! (`PhysicalCandidate::delivers_on`). +//! (`PhysicalASAPMemoCandidates::delivers_on`). use std::collections::{BTreeSet, HashMap}; @@ -255,7 +255,7 @@ impl SelectableExecutions { u: PhysicalCandidateId, ) -> Option { let producer = &physical.physical_candidates[u.0]; - if !producer.delivers_on(physical, &self.contexts[context.0]) { + if !physical.delivers_on(u, &self.contexts[context.0]) { return None; } let properties = producer.physical_properties; @@ -274,7 +274,9 @@ impl SelectableExecutions { /// The context the inputs of `consumer` run in: an `AcrossEvaluations` /// candidate's at ingestion time, the ingestion pipeline of its refresh - /// window (`LogicalASAPMemoCandidates::own_schedule`); a `TimeShift` anchored at + /// window, the one the installed plan refreshes it on + /// (`RefreshWindow::computed_from` over the candidates: `sum(TimeRange(5m))` + /// every 5 min, a 60 s chain over `TimeShift(30s)` at phase 30 s); a `TimeShift` anchored at /// `@ end()` or `@ start()` in an evaluation context, a context of its /// own per anchor, on the grid of its queries' request end (start) /// times: a request's start and end, and an evaluation, are different @@ -292,17 +294,8 @@ impl SelectableExecutions { if properties.timing() == ExecutionTiming::IngestionTime && properties.is_across_evaluations() { - return match physical - .logical_candidates - .own_schedule(candidate.logical_candidate_id) - { - Some(grid) => self.context_id( - ExecutionContext::IngestionPipeline(RefreshWindow { - width_ms: grid.period_ms, - phase_ms: grid.phase_ms, - }), - None, - ), + return match RefreshWindow::computed_from(physical, consumer.candidate) { + Some(window) => self.context_id(ExecutionContext::IngestionPipeline(window), None), None => consumer.context, }; } diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index a00db1e0a..c5616a5c3 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -75,7 +75,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use super::physical_asap_dag::{PhysicalASAPDAGValidationError, PhysicalASAPNodeId, ViewCoverage}; -use super::time_window::TimeWindow; +use super::time_window::{self, TimeIndexedPlan, TimeWindow}; use crate::ir::operator::arguments::Source; /// One materialized view of a DAG: what its stored state is, what it is @@ -172,6 +172,26 @@ impl RefreshWindow { phase_ms, }) } + + /// The refresh window of `node`, stored at ingestion time in `plan`: + /// that of the smallest window holding every window its result is + /// computed from ([`time_window::windows_computed_from`]). A + /// `SummaryTumblingWindowChain` of width `w` over shifts summing to `s` + /// is `(w, s mod w)`; any other node takes the union of the windows + /// below it (each `TimeRange`'s, and each chain's reached without + /// passing one). `None` when that window is unbounded (none below it, + /// or an `@` anchor below a window) or empty. One rule for the + /// installed plan ([`PhysicalASAPDAG::refresh_window`]), Stage 3's + /// candidates and the cost model. + /// + /// Example: `sum(TimeRange(5m))` is `(5m, 0)` whatever grid reads it; a + /// chain of 60 s windows over `TimeShift(30s)` is `(60s, 30s)`. + pub fn computed_from(plan: &P, node: P::Node) -> Option { + time_window::windows_computed_from(plan, node)? + .into_iter() + .reduce(TimeWindow::union) + .and_then(Self::of_window) + } } /// The views refreshed at ingestion that share one refresh window diff --git a/crates/types/src/plan/materialized_view/views.rs b/crates/types/src/plan/materialized_view/views.rs index 5c765e11b..835abdc39 100644 --- a/crates/types/src/plan/materialized_view/views.rs +++ b/crates/types/src/plan/materialized_view/views.rs @@ -74,12 +74,11 @@ impl PhysicalASAPDAG { .filter(|view| view.refresh == MaterializedViewRefresh::AtIngestion) .cloned() .collect(); - let refresh_windows = self.refresh_windows(views.iter().map(|view| view.node_id)); let mut groups: Vec = Vec::new(); let mut group_by_window: HashMap = HashMap::new(); let mut group_by_view: HashMap = HashMap::new(); for view in views { - let refresh_window = refresh_windows[&view.node_id]; + let refresh_window = self.refresh_window(view.node_id); let group = match refresh_window.and_then(|window| group_by_window.get(&window)) { Some(&group) => group, None => { @@ -133,34 +132,10 @@ impl PhysicalASAPDAG { .collect() } - /// The refresh window of `node` ([`RefreshWindow::of_window`]): for a - /// `SummaryTumblingWindowChain` of width `w`, `(w, s mod w)` with `s` - /// the sum of the offsets of the `TimeShift`s below it; for any other - /// node, that of the smallest window holding the windows below it (each - /// `TimeRange`'s, and each chain's reached without passing one). `None` - /// when that window is unbounded (none below it, or an `@` anchor below - /// a `TimeRange`) or empty. O(nodes + edges). + /// The refresh window of `node` ([`RefreshWindow::computed_from`] over + /// this DAG). O(nodes + edges) per window walked. pub fn refresh_window(&self, node: PhysicalASAPNodeId) -> Option { - self.refresh_windows([node])[&node] - } - - /// [`Self::refresh_window`] of each of `nodes`, by id, with the DAG - /// indexed once for all of them. - pub fn refresh_windows( - &self, - nodes: impl IntoIterator, - ) -> HashMap> { - let index = ViewIndex::new(self); - nodes - .into_iter() - .map(|node| { - let refresh_window = index - .windows_computed_from(node) - .and_then(|windows| windows.into_iter().reduce(TimeWindow::union)) - .and_then(RefreshWindow::of_window); - (node, refresh_window) - }) - .collect() + RefreshWindow::computed_from(self, node) } } diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index 4100c5463..a7c2c828e 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -285,30 +285,47 @@ pub fn windows_readers_need( Some(windows) } -/// The sum of the offsets of the `TimeShift`s directly below `node` (each -/// the only input of the node above it). `None` when one has an `@` -/// anchor, or the sum is negative or overflows. +/// The sum of the offsets of the `TimeShift`s directly below `node`, each +/// the input of the node above it. Over a plan each of these nodes has one +/// input; over Stage 3's candidates its inputs are the alternative +/// producers of its one input, and every path down must give the same +/// sum. `None` when one has an `@` anchor, the paths disagree, or the sum +/// is negative or overflows. /// /// Example: `TimeRange(5m)` over `TimeShift(1m)` over `TimeShift(1h)`: /// 3,660,000. pub fn shift_back_ms_below(plan: &P, node: P::Node) -> Option { - let only_input = |node| match plan.inputs_of(node).as_slice() { - [input] => Some(*input), - _ => None, - }; - let mut shift_back_ms = 0i64; - let mut current = node; - while let Some(input) = only_input(current) { - let Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) = plan.operator_of(input) else { - break; - }; - if shift.at.is_some() { - return None; + let mut shift_back_ms: Option = None; + let mut seen = std::collections::BTreeSet::new(); + // (a node, the offsets of the shifts walked through to it) + let mut pending = vec![(node, 0i64)]; + while let Some((current, walked_ms)) = pending.pop() { + if !seen.insert((current, walked_ms)) { + continue; + } + let inputs = plan.inputs_of(current); + // A path ends at a node with no input, or at an input that is not a + // shift. + let mut path_ends = inputs.is_empty(); + for input in inputs { + match plan.operator_of(input) { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + if shift.at.is_some() { + return None; + } + pending.push((input, walked_ms.checked_add(shift.offset_ms)?)); + } + _ => path_ends = true, + } + } + if path_ends { + if shift_back_ms.is_some_and(|sum| sum != walked_ms) { + return None; + } + shift_back_ms = Some(walked_ms); } - shift_back_ms = shift_back_ms.checked_add(shift.offset_ms)?; - current = input; } - u64::try_from(shift_back_ms).ok() + u64::try_from(shift_back_ms?).ok() } /// Down: the windows of input rows that `node`'s result is computed from: diff --git a/crates/types/src/stages/logical/group/mod.rs b/crates/types/src/stages/logical/group/mod.rs index c460544b8..cf087d8ea 100644 --- a/crates/types/src/stages/logical/group/mod.rs +++ b/crates/types/src/stages/logical/group/mod.rs @@ -90,7 +90,7 @@ pub struct GroupProperties { /// says nothing about whether rows keep arriving: a result that does /// not depend on the evaluation time is final, the same at every /// evaluation, only once its rows no longer change - /// ([`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on)). + /// ([`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on)). pub depends_on_evaluation_time: bool, /// What an aggregate group covers: a summary state, a chain, a /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact @@ -189,7 +189,7 @@ pub(super) fn derive_group_properties( /// ([`reads_time_not_pinned_by_at`]) still depends on it. Any other /// operator depends on it when an input group does. A `Scan` reads no time: /// whether its rows keep arriving is -/// [`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on)'s +/// [`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on)'s /// check. fn depends_on_evaluation_time(operator: &Operator>) -> bool { match operator { diff --git a/crates/types/src/stages/physical/candidate.rs b/crates/types/src/stages/physical/candidate.rs index 1a06cdfcc..2c345e8e2 100644 --- a/crates/types/src/stages/physical/candidate.rs +++ b/crates/types/src/stages/physical/candidate.rs @@ -2,9 +2,8 @@ //! valid $p^{\mathrm{phys}} = (\tau, \ell)$. use crate::ir::properties::ExecutionTiming; -use crate::stages::{CandidateForMemoGroupId, ExecutionContext, PhysicalASAPMemoCandidates}; +use crate::stages::CandidateForMemoGroupId; use crate::stages::{Lifetime, PhysicalProperties}; -use crate::workload::DataArrival; /// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -19,94 +18,9 @@ pub struct PhysicalCandidate { pub physical_properties: PhysicalProperties, } -impl PhysicalCandidate { - /// Whether a consumer running in `context` can read this producer (the - /// concepts doc's alignment table, section 2): true for $\ell$ = None, - /// which runs in the consumer's own context; for `OneEvaluation` only in - /// an `Evaluation` context, the one evaluation its result belongs to. - /// With $\ell$ = `AcrossEvaluations`, every execution time of the - /// context must be a time the producer has stored: for a chain - /// (`SummaryTumblingWindowChain`) of width $w$, $w$ divides an - /// evaluation grid's period and phase, or a refresh window's width and - /// phase. Any other result is stored on its group's evaluation grid - /// (`evaluation_grid_of_group`), which must hold every time of the - /// context, unless it is final: the same at every time it is read. - /// - /// Finality assumes no row arrives out of order: at evaluation time t, - /// every row with `ts <= t` has arrived and never changes (#1298 adds - /// a declared lateness). A stored result is final when its group does - /// not depend on the evaluation time - /// ([`GroupProperties::depends_on_evaluation_time`](crate::stages::GroupProperties::depends_on_evaluation_time)) - /// and the rows it read never change after it is computed: - /// - /// | Result | Delivers on | - /// |---|---| - /// | kept up to date at ingestion time (`IngestionTime`) | every context | - /// | over data at rest (`DataArrival::AtRest`) or reading no source | every context | - /// | at query time, reading a time relative to its evaluation (`rate(x[5m])`, `@ end()`, `now()`) | its own grid | - /// | at query time over arriving rows, no time bound (SQL `count(*)`) | its own grid: never final | - /// | at query time over arriving rows, a window ending at E (`sum_over_time(x[5m] @ T)`, `ts BETWEEN a AND b`) | its own grid: final only from E on, and a context carries no absolute time to compare with E | - /// - /// `DataArrival::Unknown` counts as arriving. - /// The times of a - /// `Materialized(v)` context are $v$'s own - /// ([`LogicalASAPMemoCandidates::own_schedule`](crate::stages::LogicalASAPMemoCandidates::own_schedule)); - /// when they are not known, no non-final producer delivers on it. - pub fn delivers_on( - &self, - candidates: &PhysicalASAPMemoCandidates, - context: &ExecutionContext, - ) -> bool { - match self.physical_properties.lifetime() { - Lifetime::None => true, - Lifetime::OneEvaluation(_) => matches!(context, ExecutionContext::Evaluation { .. }), - Lifetime::AcrossEvaluations(_) => { - let logical = &candidates.logical_candidates; - let group = self.logical_candidate_id.group; - let group_properties = &logical.memo.groups()[group.0 as usize].group_properties; - let final_at_every_time = !group_properties.depends_on_evaluation_time - && (self.physical_properties.timing() == ExecutionTiming::IngestionTime - || logical.memo.data_arrival() == DataArrival::AtRest - || group_properties.scanned_sources.is_empty()); - if logical - .memo - .candidate(self.logical_candidate_id) - .expression - .operator - .chain_tumbling_window_width_ms() - .is_none() - && final_at_every_time - { - return true; - } - let (period_ms, phase_ms) = match context { - ExecutionContext::Evaluation { grid } => (grid.period_ms, grid.phase_ms), - ExecutionContext::IngestionPipeline(window) => { - (window.width_ms, window.phase_ms) - } - ExecutionContext::Materialized(v) => { - let materialized = candidates.physical_candidates[v.0].logical_candidate_id; - match logical.own_schedule(materialized) { - Some(grid) => (grid.period_ms, grid.phase_ms), - None => return false, - } - } - }; - logical - .own_schedule(self.logical_candidate_id) - .is_some_and(|stored| { - stored.period_ms > 0 - && period_ms.is_multiple_of(stored.period_ms) - && phase_ms % stored.period_ms == stored.phase_ms % stored.period_ms - }) - } - } - } -} - /// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts /// doc's table, section 2). Alignment is checked per context in Stage 3 with -/// [`PhysicalCandidate::delivers_on`]. +/// [`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on). pub fn producer_satisfies_physical_requirements( producer: &PhysicalCandidate, consumer_timing: ExecutionTiming, diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs index c95744fa7..51bb7a153 100644 --- a/crates/types/src/stages/physical/execution_context.rs +++ b/crates/types/src/stages/physical/execution_context.rs @@ -1,6 +1,6 @@ //! $\kappa$: a class of executions that happen at the same times (concepts //! doc, section 5). Stage 3 indexes $x_{\kappa,v}$ and $y_{\kappa,u,c,i}$ by -//! it; [`super::PhysicalCandidate::delivers_on`] reads it, so it lives with +//! it; [`super::PhysicalASAPMemoCandidates::delivers_on`] reads it, so it lives with //! the physical candidates. use super::candidate::PhysicalCandidateId; diff --git a/crates/types/src/stages/physical/memo_candidates/mod.rs b/crates/types/src/stages/physical/memo_candidates/mod.rs index 1a8c4ce00..bc33bc8c0 100644 --- a/crates/types/src/stages/physical/memo_candidates/mod.rs +++ b/crates/types/src/stages/physical/memo_candidates/mod.rs @@ -6,11 +6,15 @@ use std::collections::HashMap; -use crate::ir::{Operator, QueryRoot}; +use crate::ir::properties::ExecutionTiming; +use crate::ir::{NonASAPOp, Operator, QueryRoot}; +use crate::plan::materialized_view::RefreshWindow; +use crate::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; use crate::stages::{ - LogicalASAPMemoCandidates, MemoGroupId, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, PrunedCandidate, + EvaluationGrid, ExecutionContext, Lifetime, LogicalASAPMemoCandidates, MemoGroupId, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PrunedCandidate, }; +use crate::workload::DataArrival; /// Stage 2's output, which Stage 3 selects from. #[derive(Debug, Clone)] @@ -111,6 +115,137 @@ impl PhysicalASAPMemoCandidates { pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { &self.admissible_edges_out_of_candidate[producer.0] } + + /// The times candidate `v`, materialized across evaluations, has a + /// stored result for, as a grid: at ingestion time the end of every + /// refresh window, by the rule the installed plan refreshes it on + /// ([`RefreshWindow::computed_from`] over these candidates, width as the + /// period); at query time its own schedule + /// ([`LogicalASAPMemoCandidates::own_schedule`]). `None` when not known. + /// + /// Example: `sum(TimeRange(5m))` kept at ingestion time is stored every + /// 5 min at phase 0, even when its group is read every 60 s; a chain of + /// 60 s windows over `TimeShift(30s)` every 60 s at phase 30 s. + pub fn stored_times(&self, v: PhysicalCandidateId) -> Option { + let candidate = &self.physical_candidates[v.0]; + match candidate.physical_properties.timing() { + ExecutionTiming::IngestionTime => { + RefreshWindow::computed_from(self, v).map(|window| EvaluationGrid { + period_ms: window.width_ms, + phase_ms: window.phase_ms, + }) + } + ExecutionTiming::QueryTime => self + .logical_candidates + .own_schedule(candidate.logical_candidate_id), + } + } + + /// Whether a consumer running in `context` can read producer `v` (the + /// concepts doc's alignment table, section 2): true for $\ell$ = None, + /// which runs in the consumer's own context; for `OneEvaluation` only in + /// an `Evaluation` context, the one evaluation its result belongs to. + /// With $\ell$ = `AcrossEvaluations`, every execution time of the + /// context must be a time the producer has a stored result for + /// ([`Self::stored_times`]): its period divides the context's period (or + /// refresh-window width) and the phases agree modulo it, unless the + /// result is final: the same at every time it is read. + /// + /// Finality assumes no row arrives out of order: at evaluation time t, + /// every row with `ts <= t` has arrived and never changes (#1298 adds + /// a declared lateness). A stored result that is not a chain is final + /// when its group does not depend on the evaluation time + /// ([`GroupProperties::depends_on_evaluation_time`](crate::stages::GroupProperties::depends_on_evaluation_time)) + /// and the rows it read never change after it is computed: + /// + /// | Result | Delivers on | + /// |---|---| + /// | kept up to date at ingestion time (`IngestionTime`) | every context | + /// | over data at rest (`DataArrival::AtRest`) or reading no source | every context | + /// | at query time, reading a time relative to its evaluation (`rate(x[5m])`, `@ end()`, `now()`) | its own grid | + /// | at query time over arriving rows, no time bound (SQL `count(*)`) | its own grid: never final | + /// | at query time over arriving rows, a window ending at E (`sum_over_time(x[5m] @ T)`, `ts BETWEEN a AND b`) | its own grid: final only from E on, and a context carries no absolute time to compare with E | + /// + /// `DataArrival::Unknown` counts as arriving. The times of a + /// `Materialized(u)` context are $u$'s stored times; when they are not + /// known, no non-final producer delivers on it. + /// + /// Example: `sum(TimeRange(5m))` kept at ingestion time delivers on an + /// evaluation grid every 5 min, not on one every 60 s. + pub fn delivers_on(&self, v: PhysicalCandidateId, context: &ExecutionContext) -> bool { + let producer = &self.physical_candidates[v.0]; + match producer.physical_properties.lifetime() { + Lifetime::None => true, + Lifetime::OneEvaluation(_) => matches!(context, ExecutionContext::Evaluation { .. }), + Lifetime::AcrossEvaluations(_) => { + let memo = &self.logical_candidates.memo; + let group_properties = + &memo.groups()[producer.logical_candidate_id.group.0 as usize].group_properties; + let final_at_every_time = !group_properties.depends_on_evaluation_time + && (producer.physical_properties.timing() == ExecutionTiming::IngestionTime + || memo.data_arrival() == DataArrival::AtRest + || group_properties.scanned_sources.is_empty()); + if self.operator(v).chain_tumbling_window_width_ms().is_none() + && final_at_every_time + { + return true; + } + let (period_ms, phase_ms) = match context { + ExecutionContext::Evaluation { grid } => (grid.period_ms, grid.phase_ms), + ExecutionContext::IngestionPipeline(window) => { + (window.width_ms, window.phase_ms) + } + ExecutionContext::Materialized(u) => match self.stored_times(*u) { + Some(grid) => (grid.period_ms, grid.phase_ms), + None => return false, + }, + }; + self.stored_times(v).is_some_and(|stored| { + stored.period_ms > 0 + && period_ms.is_multiple_of(stored.period_ms) + && phase_ms % stored.period_ms == stored.phase_ms % stored.period_ms + }) + } + } + } +} + +/// The candidates as the time walks see them: each candidate's admissible +/// producers as its inputs and admissible consumers as its readers, its +/// operator at its timing and lifetime ([`InputTimeAccess::of`]), and a +/// current-time read of a `Scan` (raw data) needing every row. The +/// producers of one input are alternatives, so a walk down over them +/// covers every plan that may select them. +impl TimeIndexedPlan for PhysicalASAPMemoCandidates { + type Node = PhysicalCandidateId; + type Child = MemoGroupId; + + fn readers_of(&self, v: PhysicalCandidateId) -> Vec { + self.admissible_edges_out_of(v) + .iter() + .map(|&out| self.edge(out).consumer_candidate_id) + .collect() + } + + fn inputs_of(&self, v: PhysicalCandidateId) -> Vec { + (0..self.input_count(v)) + .flat_map(|input| self.admissible_edges_into(v, input)) + .map(|&edge| self.edge(edge).producer_candidate_id) + .collect() + } + + fn input_time_access(&self, v: PhysicalCandidateId) -> InputTimeAccess { + let properties = self.physical_candidates[v.0].physical_properties; + InputTimeAccess::of(self.operator(v), properties.timing(), properties.lifetime()) + } + + fn current_time_read_needs_every_row(&self, v: PhysicalCandidateId) -> bool { + matches!(self.operator(v), Operator::NonASAP(NonASAPOp::Scan { .. })) + } + + fn operator_of(&self, v: PhysicalCandidateId) -> &Operator { + self.operator(v) + } } /// The output references of `query_roots`: each operator reference, in diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 5ec7c5245..51f5bfe47 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -150,6 +150,19 @@ fn candidate( } } +/// [`PhysicalASAPMemoCandidates::delivers_on`] of `producer`, added as the +/// last candidate of `candidates`. +fn delivers_on( + candidates: &PhysicalASAPMemoCandidates, + producer: &PhysicalCandidate, + context: &ExecutionContext, +) -> bool { + let mut with_producer = candidates.clone(); + with_producer.physical_candidates.push(*producer); + let v = PhysicalCandidateId(with_producer.physical_candidates.len() - 1); + with_producer.delivers_on(v, context) +} + fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { ExecutionContext::Evaluation { grid: EvaluationGrid { @@ -183,7 +196,7 @@ fn delivers_on_by_lifetime_without_windows() { (&one_evaluation, materialized, false), ] { assert_eq!( - producer.delivers_on(&candidates, &context), + delivers_on(&candidates, producer, &context), delivers, "{:?} in {context:?}", producer.physical_properties @@ -219,7 +232,7 @@ fn delivers_on_a_chain_when_its_width_divides_the_context_times() { (&ingestion_10s, ingestion(5_000, 0), false), ] { assert_eq!( - producer.delivers_on(&candidates, &context), + delivers_on(&candidates, producer, &context), delivers, "{:?} {:?} in {context:?}", producer.logical_candidate_id, @@ -251,7 +264,11 @@ fn delivers_on_a_stored_result_on_its_group_grid() { (&on_grid, ingestion(60_000, 0), true), (&without_grid, evaluation(60_000, 0), false), ] { - assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); + assert_eq!( + delivers_on(candidates, &x, &context), + delivers, + "{context:?}" + ); } } @@ -377,8 +394,8 @@ fn delivers_on_grid_and_off_it( ) -> (bool, bool) { let stored = candidate(producer, timing, Lifetime::AcrossEvaluations(MEMORY)); ( - stored.delivers_on(&examples.candidates, &evaluation(60_000, 0)), - stored.delivers_on(&examples.candidates, &evaluation(30_000, 0)), + delivers_on(&examples.candidates, &stored, &evaluation(60_000, 0)), + delivers_on(&examples.candidates, &stored, &evaluation(30_000, 0)), ) } @@ -433,7 +450,7 @@ fn an_ingestion_time_view_without_a_window_delivers_on_every_context() { ingestion(60_000, 30_000), ] { assert!( - stored.delivers_on(&examples.candidates, &context), + delivers_on(&examples.candidates, &stored, &context), "{arrival:?} {context:?}" ); } @@ -558,7 +575,7 @@ fn delivers_on_the_own_times_of_a_materialized_computation() { ), ] { assert_eq!( - producer.delivers_on(&candidates, &context), + delivers_on(&candidates, producer, &context), delivers, "{:?} in {context:?}", producer.logical_candidate_id @@ -701,3 +718,110 @@ fn new_indexes_only_admissible_edges() { .is_empty()); } } + +/// Stored at ingestion time, a result is stored at the ends of the refresh +/// windows the installed plan refreshes it on +/// ([`RefreshWindow::computed_from`]), not on its group's evaluation grid: +/// `sum(TimeRange(5m))` read every 60 s is refreshed every 5 min, so it +/// delivers on a 5 min grid but not on the 60 s one; a chain of 60 s +/// windows over `TimeShift(30s)` is refreshed at phase 30 s. +#[test] +fn an_ingestion_time_result_is_stored_on_its_refresh_window() { + let mut memo = Memo::new(&DataWorkload::default()); + let mut add = |operator: Operator, parameters| { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let candidate = CandidateForMemoGroup::new( + operator, + parameters, + OriginOfCandidateForMemoGroup::Pass1, + vec![], + ); + memo.add_candidate(group, candidate).unwrap(); + CandidateForMemoGroupId { + group, + index_in_group: 0, + } + }; + let series = add(scan("latency", &[], vec![]), None); + let last_5m = add(range(300_000, series.group), None); + let sum = add( + aggregate( + Reduction::PerEntity, + AggIntent::Sum { col: Some(1) }, + last_5m.group, + ), + None, + ); + let shifted = add(shift(30_000, None, series.group), None); + let chain = add( + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: shifted.group, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }), + Some(SketchParams::Kll { k: 269 }), + ); + let grid_60s = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, + }); + let group_count = memo.groups().len(); + let stored = Lifetime::AcrossEvaluations(MEMORY); + let candidates = PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group: vec![grid_60s; group_count], + exact_only_groups_of_query: vec![], + }, + vec![ + candidate(series, IngestionTime, Lifetime::None), + candidate(last_5m, IngestionTime, Lifetime::None), + candidate(sum, IngestionTime, stored), + candidate(shifted, IngestionTime, Lifetime::None), + candidate(chain, IngestionTime, stored), + ], + vec![ + physical_edge(0, 1, true, true), + physical_edge(1, 2, true, true), + physical_edge(0, 3, true, true), + physical_edge(3, 4, true, true), + ], + vec![], + ); + let (sum, chain) = (PhysicalCandidateId(2), PhysicalCandidateId(4)); + assert_eq!( + RefreshWindow::computed_from(&candidates, sum), + Some(RefreshWindow { + width_ms: 300_000, + phase_ms: 0 + }) + ); + assert_eq!( + RefreshWindow::computed_from(&candidates, chain), + Some(RefreshWindow { + width_ms: 60_000, + phase_ms: 30_000 + }) + ); + for (producer, context, delivers) in [ + (sum, evaluation(60_000, 0), false), + (sum, evaluation(300_000, 0), true), + (sum, evaluation(600_000, 300_000), true), + (sum, ingestion(300_000, 0), true), + (chain, evaluation(60_000, 0), false), + (chain, evaluation(60_000, 30_000), true), + (chain, evaluation(120_000, 90_000), true), + (chain, ingestion(60_000, 0), false), + ] { + assert_eq!( + candidates.delivers_on(producer, &context), + delivers, + "{producer:?} in {context:?}" + ); + } +} diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index ebc3ea718..070c46b6d 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -223,7 +223,7 @@ impl QueryRecurrence { /// known: an estimated rate, a one-off query without a time, an unknown /// recurrence, or a zero interval. Pass 2's chain widths and the evaluation grids of /// Stage 3's contexts, which - /// [`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on) + /// [`PhysicalASAPMemoCandidates::delivers_on`](crate::stages::PhysicalASAPMemoCandidates::delivers_on) /// reads, both come from it. pub fn evaluation_grid_ms(&self, range_evaluation: Option) -> Option { let times_grid_ms = match self { diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 3a16ed13e..4024f5ba1 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -363,7 +363,7 @@ fn a_top_k_ranks_values_of_its_inputs_sign() { /// `@ T` reads a fixed window, and `@ T` over an `@ end()`, `now()` or /// `time()` still depends, since those are not pinned by it. `@ end()` does /// not pin a subtree to a fixed time. A `Scan` reads no time: whether its -/// data arrives is `PhysicalCandidate::delivers_on`'s check. +/// data arrives is `PhysicalASAPMemoCandidates::delivers_on`'s check. /// /// | Group | Expression | Depends | /// |---|---|---| diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 8b3f3a847..3031b3595 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -127,7 +127,7 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- - `Selection.plan` is the selected candidate's roots with the weight requirements Stage 3 confirmed from the data facts. - A producer feeds a consumer only in a context it delivers on - (`PhysicalCandidate::delivers_on`). A result stored across evaluations + (`PhysicalASAPMemoCandidates::delivers_on`). A result stored across evaluations delivers on every context only when it is final, the same at every time it is read. This assumes no row arrives out of order: at evaluation time t, every row with `ts <= t` has arrived and never changes. A result is From 6ca90a0ace2a1ac66b2e0d31adfa8e4c7c7be1d7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:27:04 +0000 Subject: [PATCH 20/23] fix(stage3,types,cost): a query's own context for context copies; one stored-times rule; review fixes (#1207 PR 7) A query is evaluated in its own evaluation context, but the context-copy search put it in the context of its first answering execution. With an AcrossEvaluations candidate listed first that is Materialized(v), so two evaluation contexts whose queries read different exact-only groups had empty query lists, matched, and shared their variables: Q0 every 10 s and Q1 every 20 s that must read the exact quantile then selected alike, at 9 instead of 8. SelectableExecutions now records each query's evaluation context in new (evaluation_context), the copy search reads it, and selection_variables asserts that every shared z is a real (query, answering execution) pair. The copy search moves out of dominance into milp/equivalent_contexts.rs (symmetry reduction) and returns SharedVariables. Contexts are bucketed by a canonical key (candidates, kept, least need, in-edges and query signatures), so only the cost proportionality check compares two contexts; the out-edge and input-count checks it implied are a debug_assert. Dominance and the copy search share one relative tolerance (COST_RELATIVE_TOLERANCE). stored_times uses RefreshWindow::computed_from for any chain, at either timing (a query-time chain over TimeShift(30s) is stored at phase 30 s, as at ingestion time), and its group's evaluation grid for any other query-time result; own_schedule is deleted. EvaluationGrid gets From, which delivers_on uses. A Filter or Project between a chain and a TimeRange keeps the chain's tumbling-window width in the cost model, as a TimeShift does: the rule is Operator::keeps_input_window (exhaustive), which also passes the windows a TimeRange selected through them. Operator::chain_tumbling_window_width_ms and Operator::request_anchor list every operator. Also: QueryWorkloadEntry::range_span_ms at the three span sites; Memo::output_groups reads QueryRoot::operator_refs and output_references is deleted; PhysicalASAPMemoCandidates::is_query_time_scan; one ONE_SECOND_MS in asap_types::workload; docs name Stage 2's output where they said Stage 3 candidates; comments state rules, not process. --- crates/cost/src/node/usage/mod.rs | 35 +-- crates/cost/src/node/usage/tests.rs | 28 +- crates/stage3-plan-selection/src/cost.rs | 16 +- .../stage3-plan-selection/src/cost/tests.rs | 2 +- .../src/milp/dominance.rs | 216 +------------ .../src/milp/equivalent_contexts.rs | 289 ++++++++++++++++++ crates/stage3-plan-selection/src/milp/mod.rs | 13 +- .../src/milp/model/mod.rs | 26 +- .../src/milp/model/tests.rs | 204 ++++++++++--- crates/stage3-plan-selection/src/retention.rs | 4 +- .../stage3-plan-selection/src/selectable.rs | 37 ++- crates/types/src/ir/operator/node.rs | 123 +++++++- .../types/src/plan/materialized_view/mod.rs | 5 +- crates/types/src/plan/time_window.rs | 14 +- crates/types/src/stages/logical/memo/mod.rs | 15 +- .../src/stages/logical/memo_candidates.rs | 43 +-- crates/types/src/stages/mod.rs | 8 +- .../src/stages/physical/execution_context.rs | 9 +- .../stages/physical/memo_candidates/mod.rs | 79 +++-- .../stages/physical/memo_candidates/tests.rs | 102 ++++--- crates/types/src/stages/physical/mod.rs | 2 +- crates/types/src/workload/data.rs | 6 + crates/types/src/workload/mod.rs | 2 +- crates/types/src/workload/query.rs | 10 + .../design_docs/proposals/planner-layering.md | 2 +- .../asap-aware-mapping-architecture.md | 2 +- 26 files changed, 854 insertions(+), 438 deletions(-) create mode 100644 crates/stage3-plan-selection/src/milp/equivalent_contexts.rs diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index a4af944c0..8a2dccba0 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -30,7 +30,7 @@ use asap_types::workload::bounds::{ declared_non_negative, declared_series, group_count_bound, source_name, source_rows_per_ms, unchanged_source_columns, }; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, ONE_SECOND_MS}; /// One node's cost per second of wall time, with how it was measured. A /// node is one operator node of the candidate's physical plan DAG @@ -123,12 +123,14 @@ pub struct ProducerOutput<'a> { /// column) values unchanged ([`unchanged_source_columns`]). pub unchanged_columns: BTreeSet<&'a str>, /// The tumbling windows its rows span: those a `TimeRange` selects - /// from a chain (through `TimeShift`s), 1 for any other node. + /// from a chain (through the operators that keep each row's window), + /// 1 for any other node. pub tumbling_windows: u64, /// The tumbling-window width of the chain whose states its rows are - /// read from: a chain's own, kept through the `TimeShift`s over it, - /// so a `TimeRange` over them selects windows; `None` for any other - /// node. + /// read from: a chain's own, kept through the operators over it that + /// keep each row's window (`Operator::keeps_input_window`: `TimeShift`, + /// `Filter`, `Project`), so a `TimeRange` over them selects windows; + /// `None` for any other node. pub chain_tumbling_window_width_ms: Option, } @@ -151,12 +153,14 @@ pub struct NodeResourceUsageAndCost<'a> { /// column) values unchanged ([`unchanged_source_columns`]). pub unchanged_columns: BTreeSet<&'a str>, /// The tumbling windows its rows span: those a `TimeRange` selects - /// from a chain (through `TimeShift`s), 1 for any other node. + /// from a chain (through the operators that keep each row's window), + /// 1 for any other node. pub tumbling_windows: u64, /// The tumbling-window width of the chain whose states its rows are - /// read from: a chain's own, kept through the `TimeShift`s over it, - /// so a `TimeRange` over them selects windows; `None` for any other - /// node. + /// read from: a chain's own, kept through the operators over it that + /// keep each row's window (`Operator::keeps_input_window`: `TimeShift`, + /// `Filter`, `Project`), so a `TimeRange` over them selects windows; + /// `None` for any other node. pub chain_tumbling_window_width_ms: Option, /// Work of one execution, in ms on one core. pub work_ms_per_execution: f64, @@ -218,11 +222,6 @@ pub fn reads_one_batch_of( && producer.keeps_raw_rows_at_ingestion(producer_input) } -/// One second, in ms: an ingestion-time node runs over the rows arriving -/// in one second, and an ingestion-time view that maintains no window is -/// refreshed once per second. -const ONE_SECOND_MS: u64 = 1_000; - /// The time between two outputs of a node running `payload` that keeps /// rows, which its retention is divided by: a chain's tumbling window, /// else `window_ms` (the window it maintains), else one second of @@ -297,14 +296,15 @@ pub fn node_resource_usage_and_cost<'a>( .max(1); let inputs: Vec = producers.iter().map(|p| p.output).collect(); // The tumbling windows its rows span: those a `TimeRange` selects from - // a chain (through `TimeShift`s, which pass them on), 1 otherwise. + // a chain (through the operators that keep each row's window, which + // pass them on: `Operator::keeps_input_window`), 1 otherwise. let tumbling_windows = match &node.payload { Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) if chain_tumbling_window_width_ms(producers).is_some() => { selected_tumbling_windows(range, producers)? } - Payload::NonASAP(NonASAPOp::TimeShift { .. }) => producers + payload if payload.keeps_input_window() => producers .first() .map_or(1, |producer| producer.tumbling_windows), _ => 1, @@ -1023,7 +1023,8 @@ pub fn node_resource_usage_and_cost<'a>( } let chain_tumbling_window_width_ms = node.payload.chain_tumbling_window_width_ms().or_else(|| { - matches!(node.payload, Payload::NonASAP(NonASAPOp::TimeShift { .. })) + node.payload + .keeps_input_window() .then(|| chain_tumbling_window_width_ms(producers)) .flatten() }); diff --git a/crates/cost/src/node/usage/tests.rs b/crates/cost/src/node/usage/tests.rs index 27f781a29..98fadfad6 100644 --- a/crates/cost/src/node/usage/tests.rs +++ b/crates/cost/src/node/usage/tests.rs @@ -97,12 +97,15 @@ fn a_merge_over_a_chain_s_windows_outputs_one_state_per_group() { /// o11y q21's shape, `sum(rate(m[5m] offset 1h))` over a chain of 60 s /// windows: a `TimeShift` by 1 h between the chain and the 5m -/// `TimeRange`, or above the `TimeRange`, changes nothing. The range -/// selects five windows of 1000 states, and the merge over them reads -/// 5000 states into 1000, as with no shift (it read 1000, one window). +/// `TimeRange`, or above the `TimeRange`, changes nothing; nor does a +/// `Filter` or a `Project`, which keep each row's window +/// (`Operator::keeps_input_window`). The range selects five windows of +/// 1000 states, and the merge over them reads 5000 states into 1000, as +/// with nothing between (it read 1000, one window). #[test] -fn a_time_shift_over_a_chain_or_its_windows_selects_what_the_range_does() { +fn an_operator_keeping_its_input_window_over_a_chain_or_its_windows_selects_what_the_range_does() { use asap_types::ir::operator::TimeShift; + use asap_types::ir::scalar::{Predicate, ProjectItem, ScalarExpr, ScalarValue}; let settings = CostInputs::default(); let data = DataWorkload::default(); let chain = node(Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { @@ -120,6 +123,17 @@ fn a_time_shift_over_a_chain_or_its_windows_selects_what_the_range_does() { }, child: 0, })); + let filter = node(Payload::NonASAP(NonASAPOp::Filter { + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + child: 0, + })); + let project = node(Payload::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: None, + expr: ScalarExpr::Column(0), + }], + child: 0, + })); let range = node(Payload::NonASAP(NonASAPOp::TimeRange { range: Duration::from_secs(300), kind: TimeRangeKind::Range, @@ -162,8 +176,10 @@ fn a_time_shift_over_a_chain_or_its_windows_selects_what_the_range_does() { "{}", direct.0 ); - for path in [[&shift, &range], [&range, &shift]] { - assert_eq!(merge_over(&path), direct); + for keeping in [&shift, &filter, &project] { + for path in [[keeping, &range], [&range, keeping]] { + assert_eq!(merge_over(&path), direct, "{:?}", keeping.payload); + } } } diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index a6f5c61f3..3239be54b 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -60,7 +60,6 @@ use asap_cost::{ }; use asap_types::ir::operator::Source; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, Operator}; use asap_types::plan::materialized_view::RefreshWindow; use asap_types::plan::physical_asap_dag::{ PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, @@ -155,7 +154,7 @@ pub(crate) fn execution_costs( .map(|v| node_retaining(PhysicalCandidateId(v), None).retains_rows()) .collect(); let query_time_scan: Vec = (0..count) - .map(|v| is_query_time_scan(physical, PhysicalCandidateId(v))) + .map(|v| physical.is_query_time_scan(PhysicalCandidateId(v))) .collect(); let producer_of = |position: usize| { let edge = selectable.edges()[position]; @@ -480,19 +479,6 @@ pub(crate) fn execution_costs( } } -/// Whether candidate `v` is a `Scan` at query time, which reads raw data -/// in each execution. -fn is_query_time_scan(physical: &PhysicalASAPMemoCandidates, v: PhysicalCandidateId) -> bool { - physical.physical_candidates[v.0] - .physical_properties - .timing() - == ExecutionTiming::QueryTime - && matches!( - physical.operator(v), - Operator::NonASAP(NonASAPOp::Scan { .. }) - ) -} - /// $\rho$ of `execution`'s context, how often it runs, from the queries it /// serves ([`context_executions_per_second`], which `plan_cost` uses too): /// in an evaluation context, for the queries on its grid whose walk over diff --git a/crates/stage3-plan-selection/src/cost/tests.rs b/crates/stage3-plan-selection/src/cost/tests.rs index 201a7aba8..c64f88ac9 100644 --- a/crates/stage3-plan-selection/src/cost/tests.rs +++ b/crates/stage3-plan-selection/src/cost/tests.rs @@ -5,7 +5,7 @@ use std::time::Duration; use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, TimeRangeKind}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; use asap_types::stages::{ CandidateForMemoGroupId, LogicalASAPMemoCandidates, PhysicalCandidate, PhysicalEdge, PhysicalProperties, StorageMedium, diff --git a/crates/stage3-plan-selection/src/milp/dominance.rs b/crates/stage3-plan-selection/src/milp/dominance.rs index 3d5d46257..6dda54bf3 100644 --- a/crates/stage3-plan-selection/src/milp/dominance.rs +++ b/crates/stage3-plan-selection/src/milp/dominance.rs @@ -1,16 +1,14 @@ -//! Executions the selection MILP need not choose freely because another -//! does as well: one with $\ell$ = None whose `OneEvaluation` twin does -//! all it does at no more cost is left out -//! ([`dominated_by_one_evaluation_twin`]); a copy of an equivalent -//! evaluation context shares the original's variables -//! ([`copies_in_equivalent_contexts`]). +//! Executions the selection MILP need not choose because another does as +//! well: one with $\ell$ = None whose `OneEvaluation` twin does all it +//! does at no more cost is left out ([`dominated_by_one_evaluation_twin`]). -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap}; use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::{ExecutionContext, ExecutionContextId, Lifetime, PhysicalEdgeId}; +use asap_types::stages::Lifetime; use super::model::SelectionInputs; +use super::COST_RELATIVE_TOLERANCE; use crate::selectable::QueryId; /// Per execution, by position in `SelectableExecutions::executions`, @@ -36,7 +34,8 @@ pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> let executions = selectable.executions(); let edges = selectable.edges(); let count = executions.len(); - let no_more = |cost: f64, than: f64| cost <= than + 1e-12 * than.abs().max(1e-12); + let no_more = + |cost: f64, than: f64| cost <= than + COST_RELATIVE_TOLERANCE * than.abs().max(1e-12); let mut answers: Vec> = vec![BTreeSet::new(); count]; for (query, answering) in selectable.answering() { for &index in answering { @@ -106,202 +105,3 @@ pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> } dominated } - -/// Per execution, edge and query, the one whose variables it shares -/// (itself when none): a root evaluation context (no request anchor) -/// equivalent to an earlier one is a copy of it, and so are its -/// executions, edges and queries. Two such contexts are equivalent when -/// their executions are of the same candidates, kept alike, their edges -/// the same memo edges from producers that correspond (in the context, or -/// the same execution outside every evaluation context), their queries -/// pair up with the same output executions, exact-only groups and -/// range-evaluation span, and every cost in one is the same multiple -/// $\alpha$ of the other's (to a relative 1e-9, the rounding of rate times -/// cost per run). -/// -/// Exact: from an optimal plan, give both contexts the selection of the -/// one whose own cost $A$ or $B$ is lower, say $A$. The copied context -/// then costs $\alpha A \le \alpha B$, the retention $h_v$ and raw data -/// $r_s$ (each the most of their readers' needs, equal in both copies) do -/// not grow, and an execution outside them that only the other selection -/// read is left out at no cost; with $B \le A$ symmetrically. So some -/// optimum selects alike in both, and sharing their variables keeps it. -/// HiGHS no longer branches over each context's copy. -/// -/// Example: the same queries every 60 s and every 300 s (one user each): -/// the 300 s context's executions cost 1/5 of the 60 s ones' and share -/// their variables. -pub(crate) fn copies_in_equivalent_contexts( - inputs: &SelectionInputs<'_>, - kept: &[bool], -) -> (Vec, Vec, Vec) { - let selectable = inputs.selectable; - let costs = inputs.costs; - let executions = selectable.executions(); - let edges = selectable.edges(); - let mut original_of_execution: Vec = (0..executions.len()).collect(); - let mut original_of_edge: Vec = (0..edges.len()).collect(); - let mut original_of_query: Vec = (0..inputs.entries.len()).map(QueryId).collect(); - let is_root_evaluation = |context: ExecutionContextId| { - matches!( - selectable.context(context), - ExecutionContext::Evaluation { .. } - ) && selectable.request_anchor(context).is_none() - }; - let is_shared = |context: ExecutionContextId| { - !matches!( - selectable.context(context), - ExecutionContext::Evaluation { .. } - ) - }; - // Each root evaluation context's executions (ascending, so by - // candidate) and queries. - let mut executions_of: BTreeMap> = BTreeMap::new(); - for (index, execution) in executions.iter().enumerate() { - if is_root_evaluation(execution.context) { - executions_of - .entry(execution.context) - .or_default() - .push(index); - } - } - let mut outputs_of: BTreeMap> = BTreeMap::new(); - for (query, answering) in selectable.answering() { - outputs_of.entry(*query).or_default().push(answering); - } - let mut queries_of: BTreeMap> = BTreeMap::new(); - for (&query, outputs) in &outputs_of { - // A query's outputs are answered in its own context. - if let Some(&index) = outputs.first().and_then(|answering| answering.first()) { - queries_of - .entry(executions[index].context) - .or_default() - .push(query); - } - } - let edge_at: HashMap<(usize, PhysicalEdgeId, usize), usize> = edges - .iter() - .enumerate() - .map(|(position, edge)| ((edge.consumer, edge.edge, edge.producer), position)) - .collect(); - // Whether `b` is `a`'s cost times `alpha` (fixed by the first pair - // with a cost). - let proportional = |alpha: &mut Option, a: f64, b: f64| match (a == 0.0, b == 0.0) { - (true, true) => true, - (false, false) => { - let ratio = *alpha.get_or_insert(b / a); - (b - ratio * a).abs() <= 1e-9 * b.abs() - } - _ => false, - }; - let contexts: Vec = executions_of.keys().copied().collect(); - let mut originals: Vec = Vec::new(); - for © in &contexts { - let copied = originals.iter().find_map(|&original| { - let (mine, theirs) = (&executions_of[©], &executions_of[&original]); - if mine.len() != theirs.len() { - return None; - } - let mut alpha = None; - let mut mapped: HashMap = HashMap::new(); - for (&b, &a) in mine.iter().zip(theirs) { - let same = executions[a].candidate == executions[b].candidate - && kept[a] == kept[b] - && costs.least_need_ms_per_execution[a] == costs.least_need_ms_per_execution[b] - && selectable.input_count(a) == selectable.input_count(b) - && selectable.edges_out_of(a).len() == selectable.edges_out_of(b).len() - && match (&costs.per_execution[a], &costs.per_execution[b]) { - (Ok(cost_a), Ok(cost_b)) => proportional(&mut alpha, *cost_a, *cost_b), - (Err(_), Err(_)) => true, - _ => false, - }; - if !same { - return None; - } - mapped.insert(b, a); - } - let mut edges_mapped: Vec<(usize, usize)> = Vec::new(); - for (&b, &a) in mine.iter().zip(theirs) { - for input in 0..selectable.input_count(b) { - let (into_b, into_a) = ( - selectable.edges_into(b, input), - selectable.edges_into(a, input), - ); - if into_b.len() != into_a.len() { - return None; - } - for &position in into_b { - let edge = edges[position]; - let producer = match mapped.get(&edge.producer) { - Some(&producer) => producer, - None if is_shared(executions[edge.producer].context) => edge.producer, - None => return None, - }; - let twin = *edge_at.get(&(a, edge.edge, producer))?; - let same = match (costs.per_edge[twin], costs.per_edge[position]) { - (Some(cost_a), Some(cost_b)) => { - proportional(&mut alpha, cost_a, cost_b) - } - (None, None) => true, - _ => false, - }; - if !same { - return None; - } - edges_mapped.push((position, twin)); - } - } - } - // Each query of `copy` pairs with one of `original` reading the - // same output executions, exact-only groups and span. - let signature = |query: QueryId, map: &dyn Fn(usize) -> usize| { - let outputs: Vec> = outputs_of[&query] - .iter() - .map(|answering| answering.iter().map(|&index| map(index)).collect()) - .collect(); - let entry = &inputs.entries[query.0]; - ( - outputs, - inputs - .physical - .logical_candidates - .exact_only_groups_of_query - .get(query.0), - entry.range_evaluation.map(|range| range.span_ms), - ) - }; - let (my_queries, their_queries) = ( - queries_of.get(©).map_or(&[][..], Vec::as_slice), - queries_of.get(&original).map_or(&[][..], Vec::as_slice), - ); - if my_queries.len() != their_queries.len() { - return None; - } - let mut unpaired: Vec = their_queries.to_vec(); - let mut queries_mapped = Vec::new(); - for &query in my_queries { - let mine = signature(query, &|index| mapped[&index]); - let at = unpaired - .iter() - .position(|&theirs| signature(theirs, &|index| index) == mine)?; - queries_mapped.push((query, unpaired.swap_remove(at))); - } - Some((original, mapped, edges_mapped, queries_mapped)) - }); - match copied { - Some((_, mapped, edges_mapped, queries_mapped)) => { - for (b, a) in mapped { - original_of_execution[b] = a; - } - for (b, a) in edges_mapped { - original_of_edge[b] = a; - } - for (b, a) in queries_mapped { - original_of_query[b.0] = a; - } - } - None => originals.push(copy), - } - } - (original_of_execution, original_of_edge, original_of_query) -} diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs new file mode 100644 index 000000000..43c558e69 --- /dev/null +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -0,0 +1,289 @@ +//! Symmetry reduction of the selection MILP: a copy of an equivalent +//! evaluation context shares the original's variables +//! ([`copies_in_equivalent_contexts`]), so HiGHS does not branch over each +//! copy. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use asap_types::stages::{ + ExecutionContext, ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId, +}; + +use super::model::SelectionInputs; +use super::COST_RELATIVE_TOLERANCE; +use crate::selectable::QueryId; + +/// Per execution, edge (by position) and query, the one whose MILP +/// variables it takes: itself unless it is in a copy of an equivalent +/// context ([`copies_in_equivalent_contexts`]). +/// +/// Example: the same query every 60 s (Q0) and every 300 s (Q1): each +/// execution and edge of the 300 s context maps to its counterpart in the +/// 60 s one, and `original_of_query` is `[Q0, Q0]`. +pub(crate) struct SharedVariables { + /// By position in `SelectableExecutions::executions`. + pub(crate) original_of_execution: Vec, + /// By position in `SelectableExecutions::edges`. + pub(crate) original_of_edge: Vec, + /// By [`QueryId`]. + pub(crate) original_of_query: Vec, +} + +/// An execution as a root evaluation context sees it: its own, by position +/// among the context's executions, or one outside every evaluation context +/// (`Materialized(v)` or an ingestion pipeline), by position in +/// `SelectableExecutions::executions`, which every context reading it +/// shares. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +enum ExecutionInContext { + Own(usize), + Shared(usize), +} + +/// What two equivalent contexts have alike, up to their costs' common +/// multiple $\alpha$: per execution (ascending, so by candidate) its +/// candidate, whether it is kept and costed, its least retention need, and +/// per input the memo edges into it from each producer, costed or not; and +/// the sorted signatures of its queries ([`QuerySignature`]). +#[derive(Debug, PartialEq, Eq, Hash)] +struct ContextKey<'a> { + executions: Vec, + queries: Vec>, +} + +/// One execution's part of a [`ContextKey`]. +#[derive(Debug, PartialEq, Eq, Hash)] +struct ExecutionKey { + candidate: PhysicalCandidateId, + kept: bool, + costed: bool, + least_need_ms: Option, + /// Per input, each edge into it: (memo edge, producer, costed). + inputs: Vec>, +} + +/// A query's part of a [`ContextKey`]: the executions that may answer each +/// of its outputs, its exact-only groups and its range-evaluation span. +type QuerySignature<'a> = ( + Vec>, + Option<&'a BTreeSet>, + u64, +); + +/// Which executions, edges and queries share variables +/// ([`SharedVariables`]): a root evaluation context (no request anchor) +/// equivalent to an earlier one is a copy of it, and so are its +/// executions, edges and queries. Two such contexts are equivalent when +/// they have the same [`ContextKey`] (the same candidates, kept alike, the +/// same memo edges from producers that correspond, in the context or the +/// same execution outside every evaluation context, and queries that pair +/// up with the same output executions, exact-only groups and span) and +/// every cost in one is the same multiple $\alpha$ of the other's, to +/// [`COST_RELATIVE_TOLERANCE`]. Each query belongs to its own evaluation +/// context (`SelectableExecutions::evaluation_context`), wherever its +/// answering executions run. Contexts are bucketed by key, so only the +/// cost check compares two contexts. +/// +/// Exact: from an optimal plan, give both contexts the selection of the +/// one whose own cost $A$ or $B$ is lower, say $A$. The copied context +/// then costs $\alpha A \le \alpha B$, the retention $h_v$ and raw data +/// $r_s$ (each the most of their readers' needs, equal in both copies) do +/// not grow, and an execution outside them that only the other selection +/// read is left out at no cost; with $B \le A$ symmetrically. So some +/// optimum selects alike in both, and sharing their variables keeps it. +/// +/// Example: the same queries every 60 s and every 300 s (one user each): +/// the 300 s context's executions cost 1/5 of the 60 s ones' and share +/// their variables. +pub(crate) fn copies_in_equivalent_contexts( + inputs: &SelectionInputs<'_>, + kept: &[bool], +) -> SharedVariables { + let selectable = inputs.selectable; + let costs = inputs.costs; + let executions = selectable.executions(); + let edges = selectable.edges(); + let mut shared = SharedVariables { + original_of_execution: (0..executions.len()).collect(), + original_of_edge: (0..edges.len()).collect(), + original_of_query: (0..inputs.entries.len()).map(QueryId).collect(), + }; + // Each root evaluation context's executions, ascending (so by + // candidate), and each execution's position among them. + let mut executions_of: BTreeMap> = BTreeMap::new(); + let mut position_in_context: Vec> = vec![None; executions.len()]; + for (index, execution) in executions.iter().enumerate() { + if selectable.is_root_evaluation_context(execution.context) { + let own = executions_of.entry(execution.context).or_default(); + position_in_context[index] = Some(own.len()); + own.push(index); + } + } + let mut outputs_of: BTreeMap> = BTreeMap::new(); + for (query, answering) in selectable.answering() { + outputs_of.entry(*query).or_default().push(answering); + } + let mut queries_of: BTreeMap> = BTreeMap::new(); + for &query in outputs_of.keys() { + if let Some(context) = selectable.evaluation_context(query) { + queries_of.entry(context).or_default().push(query); + } + } + // `index` as `context` sees it; `None` for one in another evaluation + // context (a request context), which no copy shares. + let in_context = |context: ExecutionContextId, index: usize| { + let execution_context = executions[index].context; + match selectable.context(execution_context) { + _ if execution_context == context => { + position_in_context[index].map(ExecutionInContext::Own) + } + ExecutionContext::Evaluation { .. } => None, + ExecutionContext::IngestionPipeline(_) | ExecutionContext::Materialized(_) => { + Some(ExecutionInContext::Shared(index)) + } + } + }; + let signature = |context: ExecutionContextId, query: QueryId| -> Option> { + let outputs = outputs_of[&query] + .iter() + .map(|answering| { + answering + .iter() + .map(|&index| in_context(context, index)) + .collect::>>() + }) + .collect::>>()?; + Some(( + outputs, + inputs + .physical + .logical_candidates + .exact_only_groups_of_query + .get(query.0), + inputs.entries[query.0].range_span_ms(), + )) + }; + let key_of = |context: ExecutionContextId| -> Option> { + let executions = executions_of[&context] + .iter() + .map(|&index| { + let inputs = (0..selectable.input_count(index)) + .map(|input| { + selectable + .edges_into(index, input) + .iter() + .map(|&position| { + let edge = edges[position]; + Some(( + edge.edge, + in_context(context, edge.producer)?, + costs.per_edge[position].is_some(), + )) + }) + .collect::>>() + }) + .collect::>>()?; + Some(ExecutionKey { + candidate: executions[index].candidate, + kept: kept[index], + costed: costs.per_execution[index].is_ok(), + least_need_ms: costs.least_need_ms_per_execution[index], + inputs, + }) + }) + .collect::>>()?; + let mut queries = queries_of + .get(&context) + .into_iter() + .flatten() + .map(|&query| signature(context, query)) + .collect::>>()?; + queries.sort(); + Some(ContextKey { + executions, + queries, + }) + }; + // Whether every cost of `copy` is the same multiple of `original`'s, + // their executions and edges paired in order (equal keys). + let proportional_costs = |original: ExecutionContextId, copy: ExecutionContextId| { + let mut alpha: Option = None; + let mut proportional = |a: f64, b: f64| match (a == 0.0, b == 0.0) { + (true, true) => true, + (false, false) => { + let ratio = *alpha.get_or_insert(b / a); + (b - ratio * a).abs() <= COST_RELATIVE_TOLERANCE * b.abs() + } + _ => false, + }; + executions_of[&original] + .iter() + .zip(&executions_of[©]) + .all(|(&a, &b)| { + let own = match (&costs.per_execution[a], &costs.per_execution[b]) { + (Ok(cost_a), Ok(cost_b)) => proportional(*cost_a, *cost_b), + _ => true, + }; + own && (0..selectable.input_count(a)).all(|input| { + selectable + .edges_into(a, input) + .iter() + .zip(selectable.edges_into(b, input)) + .all(|(&edge_a, &edge_b)| { + match (costs.per_edge[edge_a], costs.per_edge[edge_b]) { + (Some(cost_a), Some(cost_b)) => proportional(cost_a, cost_b), + _ => true, + } + }) + }) + }) + }; + let mut originals_of_key: HashMap, Vec> = HashMap::new(); + for © in executions_of.keys() { + let Some(key) = key_of(copy) else { + continue; + }; + let originals = originals_of_key.entry(key).or_default(); + let Some(&original) = originals + .iter() + .find(|&&original| proportional_costs(original, copy)) + else { + originals.push(copy); + continue; + }; + for (&a, &b) in executions_of[&original].iter().zip(&executions_of[©]) { + shared.original_of_execution[b] = a; + for input in 0..selectable.input_count(b) { + for (&edge_a, &edge_b) in selectable + .edges_into(a, input) + .iter() + .zip(selectable.edges_into(b, input)) + { + shared.original_of_edge[edge_b] = edge_a; + } + } + // Implied by the key: an execution of a root context is read + // only by consumers in that context, whose in-edges the key + // pairs. + debug_assert_eq!( + selectable.edges_out_of(a).len(), + selectable.edges_out_of(b).len() + ); + } + // Equal keys: the queries pair up in signature order. + let by_signature = |context: ExecutionContextId| { + let mut queries: Vec<(QuerySignature<'_>, QueryId)> = queries_of + .get(&context) + .into_iter() + .flatten() + .map(|&query| (signature(context, query).expect("keyed"), query)) + .collect(); + queries.sort(); + queries + }; + for ((_, a), (_, b)) in by_signature(original).into_iter().zip(by_signature(copy)) { + shared.original_of_query[b.0] = a; + } + } + shared +} diff --git a/crates/stage3-plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs index 8f893b93b..32e402d83 100644 --- a/crates/stage3-plan-selection/src/milp/mod.rs +++ b/crates/stage3-plan-selection/src/milp/mod.rs @@ -4,13 +4,24 @@ //! per constraint family ([`model`]), the retention amount $h_v$ and //! constraint (R) ([`retention`]), each query's exact-only reads, //! constraint (X) ([`exact_only`]), the executions other executions -//! dominate ([`dominance`]), the graph algorithm it needs ([`graph`]), +//! dominate ([`dominance`]), the copies of equivalent contexts that share +//! variables ([`equivalent_contexts`]), the graph algorithm it needs ([`graph`]), //! the bound that leaves out executions dearer than a feasible plan before //! scaling ([`upper_bound`]), and the accuracy extension ([`accuracy`]). mod accuracy; mod dominance; +mod equivalent_contexts; pub(crate) mod exact_only; mod graph; pub(crate) mod model; mod retention; mod upper_bound; + +/// The relative tolerance to which two costs per second are taken as +/// equal (or one as no more than the other) when the MILP is reduced +/// before it is built: costs are rates times costs per run, so equal costs +/// can differ in their last bits. +/// +/// Example: 0.1 × 3.0 and 0.3 differ by about 5.6e-17, well within +/// 1e-12 of 0.3. +const COST_RELATIVE_TOLERANCE: f64 = 1e-12; diff --git a/crates/stage3-plan-selection/src/milp/model/mod.rs b/crates/stage3-plan-selection/src/milp/model/mod.rs index 99df0de6c..1394a0982 100644 --- a/crates/stage3-plan-selection/src/milp/model/mod.rs +++ b/crates/stage3-plan-selection/src/milp/model/mod.rs @@ -28,7 +28,8 @@ use asap_types::ir::operator::Source; use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; use asap_types::workload::QueryWorkloadEntry; -use super::dominance::{copies_in_equivalent_contexts, dominated_by_one_evaluation_twin}; +use super::dominance::dominated_by_one_evaluation_twin; +use super::equivalent_contexts::{copies_in_equivalent_contexts, SharedVariables}; use super::exact_only::{exact_only_constraints, exact_only_variables}; use super::graph::Components; use super::retention::{least_retention_ms, retention_constraints, retention_variables}; @@ -90,8 +91,11 @@ fn selection_variables( problem: &mut ProblemVariables, ) -> SelectionVariables { let selectable = inputs.selectable; - let (original_of_execution, original_of_edge, original_of_query) = - copies_in_equivalent_contexts(inputs, kept); + let SharedVariables { + original_of_execution, + original_of_edge, + original_of_query, + } = copies_in_equivalent_contexts(inputs, kept); let mut x: Vec = Vec::with_capacity(kept.len()); for (index, &kept) in kept.iter().enumerate() { let variable = match (original_of_execution[index], kept) { @@ -117,9 +121,25 @@ fn selection_variables( } } let mut z = BTreeMap::new(); + #[cfg(debug_assertions)] + let answering_pairs: std::collections::BTreeSet<(QueryId, usize)> = selectable + .answering() + .iter() + .flat_map(|(query, executions)| { + executions.iter().map(move |&execution| (*query, execution)) + }) + .collect(); for (query, executions) in selectable.answering() { for &execution in executions { let original = (original_of_query[query.0], original_of_execution[execution]); + // A shared z is that of a real (query, answering execution) + // pair: a copy's query pairs with an original query of the + // same output executions. + #[cfg(debug_assertions)] + debug_assert!( + answering_pairs.contains(&original), + "z of {query:?} at execution {execution} shares {original:?}, not an answering pair" + ); let variable = *z .entry(original) .or_insert_with(|| problem.add(variable().binary())); diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 4fb63ae4b..050ced554 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -551,25 +551,7 @@ fn a_query_reads_its_exact_only_groups_through_exact_candidates_only() { let state = quantile.by_algorithm[0].state; query(&mut memo, quantile.result, None); query(&mut memo, quantile.result, None); - // Every candidate, one θ of the state's. - let logical_candidates: Vec<(CandidateForMemoGroupId, Lifetime)> = (0..memo.groups().len()) - .map(|group| MemoGroupId(group as u32)) - .flat_map(|group| { - let count = match group == state { - true => 1, - false => memo.groups()[group.0 as usize].candidates.len(), - }; - (0..count).map(move |index_in_group| { - ( - CandidateForMemoGroupId { - group, - index_in_group, - }, - ONE_EVALUATION, - ) - }) - }) - .collect(); + let logical_candidates = every_candidate_one_theta_of(&memo, state); let exact_quantile = logical_candidates .iter() .position(|&(candidate, _)| candidate == own_candidate(quantile.result)) @@ -606,23 +588,13 @@ fn a_query_reads_its_exact_only_groups_through_exact_candidates_only() { } } -/// x07's shape: `quantile_over_time(0.9, m[5m])` read by Q0 every 10 s, -/// whose only paths read the KLL state (its exact quantile cannot be -/// costed), and by Q1 every 30 s, which may read every group only exactly. -/// The greedy bound leaves out of each query's trees only what that query -/// must not read, so it finds a plan (Q0 through the state, Q1 through the -/// exact quantile) and leaves out an unused scan costing 1e6/s; leaving -/// out of every query's trees what some query must not read found none. -#[test] -fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { - let mut memo = empty_memo(); - let m = scan(&mut memo); - let window = range(&mut memo, 300, m); - let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); - let state = quantile.by_algorithm[0].state; - query(&mut memo, quantile.result, None); - query(&mut memo, quantile.result, None); - let mut candidates: Vec<(CandidateForMemoGroupId, Lifetime)> = (0..memo.groups().len()) +/// Every candidate of `memo`'s groups, `OneEvaluation`, with one θ of +/// `state`'s (its first candidate only). +fn every_candidate_one_theta_of( + memo: &Memo, + state: MemoGroupId, +) -> Vec<(CandidateForMemoGroupId, Lifetime)> { + (0..memo.groups().len()) .map(|group| MemoGroupId(group as u32)) .flat_map(|group| { let count = match group == state { @@ -639,7 +611,157 @@ fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { ) }) }) + .collect() +} + +/// The shape of [`a_query_reads_its_exact_only_groups_through_exact_candidates_only`] +/// with Q0 every 10 s, which may read every group, and Q1 every 20 s, +/// which must not read the KLL state; the exact quantile's candidate costs +/// `exact_quantile_cost`, every other 1, and every cost in the 20 s +/// context is half the 10 s one's. The quantile's own candidate is also offered +/// `AcrossEvaluations` at `materialized_cost` (stored on its group's 10 s +/// grid, so it delivers on both; its inputs deliver on no `Materialized` +/// context, so it is never selected), listed first: each query's first +/// answering execution runs in `Materialized(v)`, not in the query's +/// evaluation context. Returns the candidates, the queries, the selection's +/// inputs and the exact quantile's `OneEvaluation` candidate. +fn queries_differing_in_exact_only_groups_with_a_materialized_answer( + materialized_cost: f64, + exact_quantile_cost: f64, +) -> ( + PhysicalASAPMemoCandidates, + [QueryWorkloadEntry; 2], + SelectableExecutions, + ExecutionCosts, + usize, +) { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let window = range(&mut memo, 300, m); + let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); + let state = quantile.by_algorithm[0].state; + query(&mut memo, quantile.result, None); + query(&mut memo, quantile.result, None); + let mut candidates = vec![(own_candidate(quantile.result), ACROSS_EVALUATIONS)]; + candidates.extend(every_candidate_one_theta_of(&memo, state)); + let exact_quantile = candidates + .iter() + .position(|&candidate| candidate == (own_candidate(quantile.result), ONE_EVALUATION)) + .unwrap(); + let w: Vec = (0..candidates.len()) + .map(|v| match v { + 0 => materialized_cost, + _ if v == exact_quantile => exact_quantile_cost, + _ => 1.0, + }) .collect(); + let mut physical = physical(memo, &candidates); + physical.logical_candidates.evaluation_grid_of_group[quantile.result.0 as usize] = + Some(asap_types::stages::EvaluationGrid { + period_ms: 10_000, + phase_ms: 0, + }); + physical.logical_candidates.exact_only_groups_of_query = + vec![BTreeSet::new(), BTreeSet::from([state])]; + let entries = [every(10_000), every(20_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &w, &[], &[]); + let every_20s = |index: usize| { + matches!( + selectable.context(selectable.executions()[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 20_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_20s(index) { + *cost = Ok(cost.clone().unwrap() / 2.0); + } + } + for (position, edge) in selectable.edges().iter().enumerate() { + if every_20s(edge.consumer) { + costs.per_edge[position] = costs.per_edge[position].map(|cost| cost / 2.0); + } + } + (physical, entries, selectable, costs, exact_quantile) +} + +/// [`queries_differing_in_exact_only_groups_with_a_materialized_answer`]: +/// the two evaluation contexts have the same executions at half the cost, +/// but their queries read different groups, so they are not copies +/// ([`copies_in_equivalent_contexts`]): each query is in its own +/// evaluation context (`SelectableExecutions::evaluation_context`), not +/// in that of its first answering execution, `Materialized(v)`. +#[test] +fn contexts_whose_queries_read_different_groups_are_not_copies() { + let (physical, entries, selectable, costs, _) = + queries_differing_in_exact_only_groups_with_a_materialized_answer(1.0, 100.0); + let first_answering = selectable.executions()[selectable.answering()[0].1[0]]; + assert!(matches!( + selectable.context(first_answering.context), + asap_types::stages::ExecutionContext::Materialized(_) + )); + let inputs = SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }; + let shared = copies_in_equivalent_contexts(&inputs, &vec![true; selectable.executions().len()]); + assert!(shared + .original_of_execution + .iter() + .enumerate() + .all(|(index, &original)| original == index)); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(1)]); +} + +/// [`queries_differing_in_exact_only_groups_with_a_materialized_answer`] +/// with the exact quantile costing 4 (2 in the 20 s context), every other +/// candidate 1: the optimum reads the KLL readout for Q0 (its five +/// candidates, 5) and the exact quantile for Q1 (0.5 + 0.5 + 2), 8. +/// Sharing the two contexts' variables would make Q0 read the exact +/// quantile too, at 9. +#[test] +fn queries_reading_different_groups_select_differently_in_contexts_alike() { + let (physical, entries, selectable, costs, exact_quantile) = + queries_differing_in_exact_only_groups_with_a_materialized_answer(1.0, 4.0); + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + let outputs: Vec = solved + .outputs + .iter() + .map(|execution| execution.candidate.0) + .collect(); + assert_ne!(outputs[0], exact_quantile, "{outputs:?}"); + assert_eq!(outputs[1], exact_quantile, "{outputs:?}"); + assert!( + (solved.cost_per_second_without_request_overhead - 8.0).abs() < 1e-9, + "{}", + solved.cost_per_second_without_request_overhead + ); +} + +/// x07's shape: `quantile_over_time(0.9, m[5m])` read by Q0 every 10 s, +/// whose only paths read the KLL state (its exact quantile cannot be +/// costed), and by Q1 every 30 s, which may read every group only exactly. +/// The greedy bound leaves out of each query's trees only what that query +/// must not read, so it finds a plan (Q0 through the state, Q1 through the +/// exact quantile) and leaves out an unused scan costing 1e6/s; leaving +/// out of every query's trees what some query must not read found none. +#[test] +fn the_bound_leaves_out_of_each_query_only_what_it_must_not_read() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let window = range(&mut memo, 300, m); + let quantile = quantile_over_time_readouts(&mut memo, window, 0.9, &[SketchAlgorithm::Kll]); + let state = quantile.by_algorithm[0].state; + query(&mut memo, quantile.result, None); + query(&mut memo, quantile.result, None); + let mut candidates = every_candidate_one_theta_of(&memo, state); let exact_quantile = candidates .iter() .position(|&(candidate, _)| candidate == own_candidate(quantile.result)) @@ -825,7 +947,11 @@ fn a_context_equivalent_to_another_shares_its_variables() { }; copies_in_equivalent_contexts(&inputs, &kept) }; - let (original_of_execution, original_of_edge, original_of_query) = copies(&costs); + let SharedVariables { + original_of_execution, + original_of_edge, + original_of_query, + } = copies(&costs); for (index, &original) in original_of_execution.iter().enumerate() { match every_20s(index) { true => { @@ -857,8 +983,8 @@ fn a_context_equivalent_to_another_shares_its_variables() { .find(|&index| every_20s(index)) .unwrap(); costs.per_execution[first_of_20s] = Ok(10.0); - let (original_of_execution, _, _) = copies(&costs); - assert!(original_of_execution + assert!(copies(&costs) + .original_of_execution .iter() .enumerate() .all(|(index, &original)| original == index)); diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 829a4cf9d..0ace2bb4c 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -36,8 +36,8 @@ pub(crate) fn edge_retention_need_ms( memo.groups()[group.0 as usize] .reached_by_queries() .iter() - .filter_map(|&query| entries.get(query)?.range_evaluation) - .map(|range| range.span_ms) + .filter_map(|&query| entries.get(query)) + .map(QueryWorkloadEntry::range_span_ms) .max() .unwrap_or(0) }) diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index 37cfd3a2c..ea635ab99 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -45,12 +45,6 @@ use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueryId(pub usize); -/// How far back a query reads the output it reads directly: its -/// range-evaluation span (0 without one). -fn output_retention_need_ms(entry: &QueryWorkloadEntry) -> u64 { - entry.range_evaluation.map_or(0, |range| range.span_ms) -} - /// What reads an execution's rows back ([`SelectableExecutions::reads_back`]): a selectable edge /// out of it, by position in [`SelectableExecutions::edges`], or a query /// output it may answer. @@ -119,6 +113,15 @@ pub(crate) struct SelectableExecutions { /// `Memo::output_groups` order, with the executions, by position, that may /// answer it: $z_{q,v}$ exists only for them. answering: Vec<(QueryId, Vec)>, + /// Per query, by [`QueryId`], the evaluation context it is evaluated + /// in: `Evaluation { grid: evaluation_grid(entry) }`, no request + /// anchor; `None` for a query with no output reference. Its answering + /// executions may run elsewhere (an `AcrossEvaluations` one in its own + /// `Materialized(v)`), so this is not read off them. + /// + /// Example: a query every 60 s is evaluated in the context of grid + /// 60 s at phase 0. + evaluation_context_of_query: Vec>, } impl SelectableExecutions { @@ -144,6 +147,7 @@ impl SelectableExecutions { edges_into: Vec::new(), edges_out_of: Vec::new(), answering: Vec::new(), + evaluation_context_of_query: vec![None; entries.len()], }; let mut reached: BTreeSet = BTreeSet::new(); let mut pending: Vec = Vec::new(); @@ -155,6 +159,7 @@ impl SelectableExecutions { }, None, ); + selectable.evaluation_context_of_query[query.0] = Some(context); let mut executions = Vec::new(); for (v, candidate) in physical.physical_candidates.iter().enumerate() { if candidate.logical_candidate_id.group != group @@ -359,6 +364,24 @@ impl SelectableExecutions { } } + /// The evaluation context query `query` is evaluated in + /// ([`Self::evaluation_context_of_query`]); `None` for a query with no + /// output reference. + pub(crate) fn evaluation_context(&self, query: QueryId) -> Option { + self.evaluation_context_of_query + .get(query.0) + .copied() + .flatten() + } + + /// Whether the context with id `id` is a root evaluation context: a + /// query's own (`Evaluation`, no request anchor), not a request + /// context, `Materialized(v)` or an ingestion pipeline. + pub(crate) fn is_root_evaluation_context(&self, id: ExecutionContextId) -> bool { + matches!(self.contexts[id.0], ExecutionContext::Evaluation { .. }) + && self.request_anchor_of_context[id.0].is_none() + } + /// The context with id `id`. pub(crate) fn context(&self, id: ExecutionContextId) -> ExecutionContext { self.contexts[id.0] @@ -416,7 +439,7 @@ impl SelectableExecutions { |(position, (edge, &need_ms))| (edge.producer, Reader::Edge(position), need_ms), ); let outputs = self.answering().iter().flat_map(move |(query, answering)| { - let need_ms = output_retention_need_ms(&entries[query.0]); + let need_ms = entries[query.0].range_span_ms(); answering .iter() .map(move |&index| (index, Reader::Output(*query), Some(need_ms))) diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 23bcc7699..40945fe0d 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -61,14 +61,49 @@ impl Operator { } } - /// Its tumbling-window width when it is a `SummaryTumblingWindowChain`. + /// Its tumbling-window width when it is a `SummaryTumblingWindowChain`, + /// in ms; `None` for every other operator. E.g. a chain of 1-minute + /// windows: `Some(60_000)`; `TimeRange(5m)`: `None`. pub fn chain_tumbling_window_width_ms(&self) -> Option { match self { Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. }) => Some(*tumbling_window_width_ms), - _ => None, + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => None, } } @@ -79,9 +114,89 @@ impl Operator { match self { Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => match shift.at { Some(at @ (AtModifier::End | AtModifier::Start)) => Some(at), - _ => None, + Some(AtModifier::Timestamp(_)) | None => None, }, - _ => None, + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => None, + } + } + + /// Whether each output row keeps its input row's time window, so the + /// window its input covers is also its own: `Filter` and `Project` + /// keep rows and times; `TimeShift` moves every row by one offset but + /// keeps the window's length. E.g. `Filter <- TimeRange(5m)` covers + /// 5 minutes, and `Project <- chain of 1-minute windows` holds + /// 1-minute windows; `Aggregate <- TimeRange(5m)` combines rows, so + /// `false`. + pub fn keeps_input_window(&self) -> bool { + match self { + Operator::NonASAP( + NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::TimeShift { .. }, + ) => true, + Operator::NonASAP( + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::PromqlVectorFromScalar(..) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. }, + ) + | Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. }, + ) => false, } } } diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index c5616a5c3..3d9f8d7f1 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -181,8 +181,9 @@ impl RefreshWindow { /// below it (each `TimeRange`'s, and each chain's reached without /// passing one). `None` when that window is unbounded (none below it, /// or an `@` anchor below a window) or empty. One rule for the - /// installed plan ([`PhysicalASAPDAG::refresh_window`]), Stage 3's - /// candidates and the cost model. + /// installed plan ([`PhysicalASAPDAG::refresh_window`]), Stage 2's + /// output (`PhysicalASAPMemoCandidates`, which Stage 3 selects from) + /// and the cost model. /// /// Example: `sum(TimeRange(5m))` is `(5m, 0)` whatever grid reads it; a /// chain of 60 s windows over `TimeShift(30s)` is `(60s, 30s)`. diff --git a/crates/types/src/plan/time_window.rs b/crates/types/src/plan/time_window.rs index a7c2c828e..a9ab9a4cf 100644 --- a/crates/types/src/plan/time_window.rs +++ b/crates/types/src/plan/time_window.rs @@ -7,8 +7,8 @@ //! 2. [`InputTimeAccess`]: how one node reads its time-indexed input, //! which rows ([`InputRows`]) and how much earlier its output rows are //! than those ([`InputTimeAccess::shifts_back_ms`]). -//! 3. [`TimeIndexedPlan`]: a plan the walks run over (plan nodes, Stage 3 -//! candidates or executor nodes): each node's readers, inputs, input +//! 3. [`TimeIndexedPlan`]: a plan the walks run over (plan nodes, Stage 2's +//! output `PhysicalASAPMemoCandidates` or executor nodes): each node's readers, inputs, input //! time access and operator, and which nodes are raw data. //! 4. The walks: up from a time-indexed producer (raw data, a chain, //! results stored per evaluation time) through its readers, the window @@ -158,8 +158,8 @@ impl InputTimeAccess { } } -/// A plan the time walks run over: plan nodes, Stage 3 candidates or -/// executor nodes. +/// A plan the time walks run over: plan nodes, Stage 2's output +/// (`PhysicalASAPMemoCandidates`) or executor nodes. pub trait TimeIndexedPlan { type Node: Copy + Ord; /// The child type of the operators it holds. @@ -287,9 +287,9 @@ pub fn windows_readers_need( /// The sum of the offsets of the `TimeShift`s directly below `node`, each /// the input of the node above it. Over a plan each of these nodes has one -/// input; over Stage 3's candidates its inputs are the alternative -/// producers of its one input, and every path down must give the same -/// sum. `None` when one has an `@` anchor, the paths disagree, or the sum +/// input; over Stage 2's output (`PhysicalASAPMemoCandidates`) its inputs +/// are the alternative producers of its one input, and every path down +/// must give the same sum. `None` when one has an `@` anchor, the paths disagree, or the sum /// is negative or overflows. /// /// Example: `TimeRange(5m)` over `TimeShift(1m)` over `TimeShift(1h)`: diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 7d733fa2a..7d52645b2 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -181,13 +181,16 @@ impl Memo { /// operator root, or one operator of its scalar root, reads), /// deduplicated, ascending. pub fn output_groups(&self) -> Vec<(usize, MemoGroupId)> { - let roots: Vec> = self - .queries + self.queries .iter() - .map(|query| query.root.clone()) - .collect(); - crate::stages::output_references(&roots) - .into_iter() + .enumerate() + .flat_map(|(index, query)| { + query + .root + .operator_refs() + .into_iter() + .map(move |&group| (index, group)) + }) .collect::>() .into_iter() .collect() diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 5bb674155..537fbf4d6 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -4,17 +4,38 @@ use std::collections::BTreeSet; +use crate::plan::materialized_view::RefreshWindow; + use crate::stages::{ CandidateForMemoGroupId, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate, }; -/// The evaluation times `phase_ms + k · period_ms`. +/// The times `phase_ms + k · period_ms`: the evaluation times of a +/// context, or the times a result materialized across evaluations is +/// stored at (`PhysicalASAPMemoCandidates::stored_times`). +/// +/// Example: a query every 60 s is evaluated on `{ period_ms: 60_000, +/// phase_ms: 0 }`; a chain of 60 s windows over `TimeShift(30s)` is stored +/// on `{ period_ms: 60_000, phase_ms: 30_000 }`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct EvaluationGrid { pub period_ms: u64, pub phase_ms: u64, } +/// The times a refresh window ends at: every `width_ms`, at `phase_ms`. +/// +/// Example: `RefreshWindow { width_ms: 300_000, phase_ms: 0 }`, the +/// refresh of `sum(TimeRange(5m))`, ends every 5 min at phase 0. +impl From for EvaluationGrid { + fn from(window: RefreshWindow) -> Self { + EvaluationGrid { + period_ms: window.width_ms, + phase_ms: window.phase_ms, + } + } +} + /// Stage 1, Pass 2's output. #[derive(Debug, Clone)] pub struct LogicalASAPMemoCandidates { @@ -72,24 +93,4 @@ impl LogicalASAPMemoCandidates { .parameters() .is_some() } - - /// The times `candidate` is stored at when materialized across - /// evaluations, on its own schedule: every window end of a chain (its - /// width at phase 0), else its group's evaluation grid - /// ([`Self::evaluation_grid_of_group`]); `None` when neither is known. - pub fn own_schedule(&self, candidate: CandidateForMemoGroupId) -> Option { - match self - .memo - .candidate(candidate) - .expression - .operator - .chain_tumbling_window_width_ms() - { - Some(width_ms) => Some(EvaluationGrid { - period_ms: width_ms, - phase_ms: 0, - }), - None => self.evaluation_grid_of_group[candidate.group.0 as usize], - } - } } diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 4e0e91ad1..9b3ba2678 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -22,10 +22,10 @@ pub use logical::{ ValueSign, }; pub use physical::{ - anchor_is_evaluation_time, output_references, producer_satisfies_physical_requirements, - queries_of_context, query_reads, ExecutionContext, ExecutionContextId, Lifetime, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, - PhysicalEdgeId, PhysicalProperties, PhysicalPropertiesError, StorageMedium, + anchor_is_evaluation_time, producer_satisfies_physical_requirements, queries_of_context, + query_reads, ExecutionContext, ExecutionContextId, Lifetime, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, PhysicalProperties, + PhysicalPropertiesError, StorageMedium, }; pub use pruning::{ PruneCause, PruneReason, PrunedCandidate, PrunedLogicalCandidate, RootPruneReason, diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs index 51bb7a153..863fff596 100644 --- a/crates/types/src/stages/physical/execution_context.rs +++ b/crates/types/src/stages/physical/execution_context.rs @@ -7,7 +7,7 @@ use super::candidate::PhysicalCandidateId; use crate::ir::operator::AtModifier; use crate::plan::materialized_view::RefreshWindow; use crate::stages::EvaluationGrid; -use crate::workload::QueryWorkloadEntry; +use crate::workload::{QueryWorkloadEntry, ONE_SECOND_MS}; use std::collections::{BTreeSet, HashMap, HashSet}; /// Index of an [`ExecutionContext`] in the workload's list of distinct @@ -35,7 +35,7 @@ impl ExecutionContext { /// materialized: each second of arriving rows, which it computes once /// for every ingestion-time view of every refresh window that reads it. pub const EVERY_ARRIVING_ROW: Self = Self::IngestionPipeline(RefreshWindow { - width_ms: 1_000, + width_ms: ONE_SECOND_MS, phase_ms: 0, }); } @@ -48,8 +48,9 @@ impl ExecutionContext { /// every evaluation of the grid serves the range queries' request ends /// too). Otherwise it runs once per request, in a context of its own. /// -/// `@ start()` of an instant query is its evaluation time too; it keeps -/// its request context here, as the user decided for `@ end()` only. +/// `@ start()` of an instant query is its evaluation time too, but it +/// keeps a request context of its own: only `@ end()` is taken as the +/// evaluation time. /// /// Example: `sum_over_time(m[5m] @ end())` and `sum_over_time(m[5m])`, /// both instant every 60 s: true, so `m[5m]` is computed once per minute diff --git a/crates/types/src/stages/physical/memo_candidates/mod.rs b/crates/types/src/stages/physical/memo_candidates/mod.rs index bc33bc8c0..c52541a0f 100644 --- a/crates/types/src/stages/physical/memo_candidates/mod.rs +++ b/crates/types/src/stages/physical/memo_candidates/mod.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use crate::ir::properties::ExecutionTiming; -use crate::ir::{NonASAPOp, Operator, QueryRoot}; +use crate::ir::{NonASAPOp, Operator}; use crate::plan::materialized_view::RefreshWindow; use crate::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; use crate::stages::{ @@ -116,28 +116,42 @@ impl PhysicalASAPMemoCandidates { &self.admissible_edges_out_of_candidate[producer.0] } + /// Whether candidate `v` is a `Scan` run at query time, which reads raw + /// data in each execution (a current-time read of it needs every row, + /// [`TimeIndexedPlan::current_time_read_needs_every_row`]). + /// + /// Example: a query-time `Scan` of `latency`: true; the same scan kept + /// at ingestion time, or a `TimeRange(5m)` over it: false. + pub fn is_query_time_scan(&self, v: PhysicalCandidateId) -> bool { + self.physical_candidates[v.0].physical_properties.timing() == ExecutionTiming::QueryTime + && self.current_time_read_needs_every_row(v) + } + /// The times candidate `v`, materialized across evaluations, has a - /// stored result for, as a grid: at ingestion time the end of every + /// stored result for, as a grid. A chain, at either timing, and any + /// result kept at ingestion time are stored at the end of every /// refresh window, by the rule the installed plan refreshes it on /// ([`RefreshWindow::computed_from`] over these candidates, width as the - /// period); at query time its own schedule - /// ([`LogicalASAPMemoCandidates::own_schedule`]). `None` when not known. + /// period); any other result computed at query time on its group's + /// evaluation grid + /// ([`LogicalASAPMemoCandidates::evaluation_grid_of_group`]). `None` + /// when not known. /// /// Example: `sum(TimeRange(5m))` kept at ingestion time is stored every - /// 5 min at phase 0, even when its group is read every 60 s; a chain of - /// 60 s windows over `TimeShift(30s)` every 60 s at phase 30 s. + /// 5 min at phase 0, even when its group is read every 60 s; computed + /// at query time and read every 60 s, every 60 s. A chain of 60 s + /// windows over `TimeShift(30s)` every 60 s at phase 30 s. pub fn stored_times(&self, v: PhysicalCandidateId) -> Option { let candidate = &self.physical_candidates[v.0]; + let is_chain = self.operator(v).chain_tumbling_window_width_ms().is_some(); match candidate.physical_properties.timing() { - ExecutionTiming::IngestionTime => { - RefreshWindow::computed_from(self, v).map(|window| EvaluationGrid { - period_ms: window.width_ms, - phase_ms: window.phase_ms, - }) + ExecutionTiming::QueryTime if !is_chain => { + self.logical_candidates.evaluation_grid_of_group + [candidate.logical_candidate_id.group.0 as usize] + } + ExecutionTiming::QueryTime | ExecutionTiming::IngestionTime => { + RefreshWindow::computed_from(self, v).map(EvaluationGrid::from) } - ExecutionTiming::QueryTime => self - .logical_candidates - .own_schedule(candidate.logical_candidate_id), } } @@ -152,8 +166,7 @@ impl PhysicalASAPMemoCandidates { /// result is final: the same at every time it is read. /// /// Finality assumes no row arrives out of order: at evaluation time t, - /// every row with `ts <= t` has arrived and never changes (#1298 adds - /// a declared lateness). A stored result that is not a chain is final + /// every row with `ts <= t` has arrived and never changes. A stored result that is not a chain is final /// when its group does not depend on the evaluation time /// ([`GroupProperties::depends_on_evaluation_time`](crate::stages::GroupProperties::depends_on_evaluation_time)) /// and the rows it read never change after it is computed: @@ -190,20 +203,18 @@ impl PhysicalASAPMemoCandidates { { return true; } - let (period_ms, phase_ms) = match context { - ExecutionContext::Evaluation { grid } => (grid.period_ms, grid.phase_ms), - ExecutionContext::IngestionPipeline(window) => { - (window.width_ms, window.phase_ms) - } - ExecutionContext::Materialized(u) => match self.stored_times(*u) { - Some(grid) => (grid.period_ms, grid.phase_ms), + let times = match *context { + ExecutionContext::Evaluation { grid } => grid, + ExecutionContext::IngestionPipeline(window) => EvaluationGrid::from(window), + ExecutionContext::Materialized(u) => match self.stored_times(u) { + Some(grid) => grid, None => return false, }, }; self.stored_times(v).is_some_and(|stored| { stored.period_ms > 0 - && period_ms.is_multiple_of(stored.period_ms) - && phase_ms % stored.period_ms == stored.phase_ms % stored.period_ms + && times.period_ms.is_multiple_of(stored.period_ms) + && times.phase_ms % stored.period_ms == stored.phase_ms % stored.period_ms }) } } @@ -248,23 +259,5 @@ impl TimeIndexedPlan for PhysicalASAPMemoCandidates { } } -/// The output references of `query_roots`: each operator reference, in -/// order (a scalar root's in `map_operator_refs` order), with the index of -/// the query whose root it is. -pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, C)> { - let mut references = Vec::new(); - for (query, root) in query_roots.iter().enumerate() { - match root { - QueryRoot::Operator(reference) => references.push((query, reference.clone())), - QueryRoot::Scalar(expression) => { - expression.map_operator_refs(&mut |reference| { - references.push((query, reference.clone())) - }); - } - } - } - references -} - #[cfg(test)] mod tests; diff --git a/crates/types/src/stages/physical/memo_candidates/tests.rs b/crates/types/src/stages/physical/memo_candidates/tests.rs index 51f5bfe47..14a3ee2fa 100644 --- a/crates/types/src/stages/physical/memo_candidates/tests.rs +++ b/crates/types/src/stages/physical/memo_candidates/tests.rs @@ -723,10 +723,12 @@ fn new_indexes_only_admissible_edges() { /// windows the installed plan refreshes it on /// ([`RefreshWindow::computed_from`]), not on its group's evaluation grid: /// `sum(TimeRange(5m))` read every 60 s is refreshed every 5 min, so it -/// delivers on a 5 min grid but not on the 60 s one; a chain of 60 s -/// windows over `TimeShift(30s)` is refreshed at phase 30 s. +/// delivers on a 5 min grid but not on the 60 s one. A chain of 60 s +/// windows over `TimeShift(30s)` is stored at phase 30 s at either timing +/// (its windows end 30 s before the evaluation time); computed at query +/// time, `sum(TimeRange(5m))` is stored on its group's 60 s grid. #[test] -fn an_ingestion_time_result_is_stored_on_its_refresh_window() { +fn a_stored_chain_and_an_ingestion_time_result_are_stored_on_their_refresh_window() { let mut memo = Memo::new(&DataWorkload::default()); let mut add = |operator: Operator, parameters| { let group = memo.find_or_add_group(operator.clone()).unwrap(); @@ -770,58 +772,70 @@ fn an_ingestion_time_result_is_stored_on_its_refresh_window() { }); let group_count = memo.groups().len(); let stored = Lifetime::AcrossEvaluations(MEMORY); - let candidates = PhysicalASAPMemoCandidates::new( - LogicalASAPMemoCandidates { - memo, - edges: vec![], - pruned: vec![], - evaluation_grid_of_group: vec![grid_60s; group_count], - exact_only_groups_of_query: vec![], - }, - vec![ - candidate(series, IngestionTime, Lifetime::None), - candidate(last_5m, IngestionTime, Lifetime::None), - candidate(sum, IngestionTime, stored), - candidate(shifted, IngestionTime, Lifetime::None), - candidate(chain, IngestionTime, stored), - ], - vec![ - physical_edge(0, 1, true, true), - physical_edge(1, 2, true, true), - physical_edge(0, 3, true, true), - physical_edge(3, 4, true, true), - ], - vec![], - ); + let candidates_at = |timing: ExecutionTiming| { + PhysicalASAPMemoCandidates::new( + LogicalASAPMemoCandidates { + memo: memo.clone(), + edges: vec![], + pruned: vec![], + evaluation_grid_of_group: vec![grid_60s; group_count], + exact_only_groups_of_query: vec![], + }, + vec![ + candidate(series, timing, Lifetime::None), + candidate(last_5m, timing, Lifetime::None), + candidate(sum, timing, stored), + candidate(shifted, timing, Lifetime::None), + candidate(chain, timing, stored), + ], + vec![ + physical_edge(0, 1, true, true), + physical_edge(1, 2, true, true), + physical_edge(0, 3, true, true), + physical_edge(3, 4, true, true), + ], + vec![], + ) + }; let (sum, chain) = (PhysicalCandidateId(2), PhysicalCandidateId(4)); + let ingestion_time = candidates_at(IngestionTime); assert_eq!( - RefreshWindow::computed_from(&candidates, sum), + RefreshWindow::computed_from(&ingestion_time, sum), Some(RefreshWindow { width_ms: 300_000, phase_ms: 0 }) ); - assert_eq!( - RefreshWindow::computed_from(&candidates, chain), - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 30_000 - }) - ); - for (producer, context, delivers) in [ - (sum, evaluation(60_000, 0), false), - (sum, evaluation(300_000, 0), true), - (sum, evaluation(600_000, 300_000), true), - (sum, ingestion(300_000, 0), true), - (chain, evaluation(60_000, 0), false), - (chain, evaluation(60_000, 30_000), true), - (chain, evaluation(120_000, 90_000), true), - (chain, ingestion(60_000, 0), false), + let chain_stored = EvaluationGrid { + period_ms: 60_000, + phase_ms: 30_000, + }; + let query_time = candidates_at(QueryTime); + assert_eq!(ingestion_time.stored_times(chain), Some(chain_stored)); + assert_eq!(query_time.stored_times(chain), Some(chain_stored)); + assert_eq!(query_time.stored_times(sum), grid_60s); + for (candidates, producer, context, delivers) in [ + (&ingestion_time, sum, evaluation(60_000, 0), false), + (&ingestion_time, sum, evaluation(300_000, 0), true), + (&ingestion_time, sum, evaluation(600_000, 300_000), true), + (&ingestion_time, sum, ingestion(300_000, 0), true), + (&ingestion_time, chain, evaluation(60_000, 0), false), + (&ingestion_time, chain, evaluation(60_000, 30_000), true), + (&ingestion_time, chain, evaluation(120_000, 90_000), true), + (&ingestion_time, chain, ingestion(60_000, 0), false), + (&query_time, sum, evaluation(60_000, 0), true), + (&query_time, sum, evaluation(60_000, 30_000), false), + (&query_time, chain, evaluation(60_000, 0), false), + (&query_time, chain, evaluation(60_000, 30_000), true), + (&query_time, chain, evaluation(120_000, 90_000), true), ] { assert_eq!( candidates.delivers_on(producer, &context), delivers, - "{producer:?} in {context:?}" + "{producer:?} in {context:?} at {:?}", + candidates.physical_candidates[producer.0] + .physical_properties + .timing() ); } } diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index c8d45dd71..c07a8627a 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -22,5 +22,5 @@ pub use execution_context::{ anchor_is_evaluation_time, queries_of_context, query_reads, ExecutionContext, ExecutionContextId, }; -pub use memo_candidates::{output_references, PhysicalASAPMemoCandidates}; +pub use memo_candidates::PhysicalASAPMemoCandidates; pub use properties::{Lifetime, PhysicalProperties, PhysicalPropertiesError, StorageMedium}; diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index 42d16a762..93385c36e 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -7,6 +7,12 @@ use serde::{Deserialize, Serialize}; use super::{DurationMs, WorkloadError}; +/// One second, in ms: the unit of every per-second rate +/// ([`DataArrival::rows_per_second`]). An ingestion-time node runs over +/// the rows arriving in one second, and an ingestion-time view that +/// maintains no window is refreshed once per second. +pub const ONE_SECOND_MS: u64 = 1_000; + /// How the queried data arrives: already stored, still arriving, or both. /// This is independent of whether queries repeat. /// diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index 2af8e5966..cff20a85c 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -11,7 +11,7 @@ mod query; pub(crate) use data::series_facts; pub use data::{ DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, - StreamNorms, + StreamNorms, ONE_SECOND_MS, }; pub use planning::{PlanningWorkload, WorkloadError}; pub use query::{ diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index 070c46b6d..c1aeef8be 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -427,6 +427,16 @@ impl QueryWorkloadEntry { .unwrap_or(1) } + /// How far back from a request's end its evaluations reach, in ms: its + /// range-evaluation span, 0 for one evaluation per run. The output it + /// reads directly is read back that far. + /// + /// Example: a range request over the last hour at 60 s steps: + /// 3,600,000 ms; an instant query: 0. + pub fn range_span_ms(&self) -> u64 { + self.range_evaluation.map_or(0, |range| range.span_ms) + } + /// The grid, in ms, of the times its requests anchor `@ end()` or /// `@ start()` (`at`) at: its request end times, shifted back by its /// range-evaluation span for `@ start()`; 1 ms when not known. diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 8017eeaa1..a112954eb 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -511,7 +511,7 @@ candidates is in `crates/stage3-plan-selection/src/` (`lib.rs`, `selectable.rs`, | Pass 1 | `pass1_memo` → `Memo`, the one entry: inside it, the computation targets (walking the memo groups, less the summaries the deployment does not support; `ComputationTarget` is Pass 1's private working type), then each target's ways to compute added as candidates of the groups, a summary with one build per configuration of its `parameter_list` | `stage1-pass1-per-computation-candidates/src/lib.rs`; `computation_targets.rs`; `candidates/` (`mod.rs`, `pending.rs`, `measure_output.rs`); `summary_realization.rs` builds the summary inputs (the sketch algorithms that serve an intent, each row's update) | | Memo types | `Memo`, `MemoGroup`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges; then each group's evaluation grid and the groups each query may read only exactly | `stage1-pass2-reuse-candidates/src/` (`lib.rs`, `rules/{candidates.rs,rollup/,topk.rs,chains.rs}`, `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs`, `evaluation_grids.rs`; the readout levels: `asap_accuracy::readout_levels`; the groups each query may read only exactly: `asap_accuracy::ErrorMetricsOfMemo::exact_only_groups_of_query`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | -| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `retention.rs` for $h_v$ and (R), `exact_only.rs` for each query's exact-only reads $a_{q,e}$ and (X), `dominance.rs` for executions their `OneEvaluation` twin dominates, `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | +| Production memo build and search | The facade's `run_planning` builds Stage 0's memo (`memo_of_queries`) and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the executions and edges it may select (`selectable.rs`), their costs (`cost.rs`) and the selection MILP over execution contexts (`milp/`: `model/`, `retention.rs` for $h_v$ and (R), `exact_only.rs` for each query's exact-only reads $a_{q,e}$ and (X), `dominance.rs` for executions their `OneEvaluation` twin dominates, `equivalent_contexts.rs` for the copies of an equivalent evaluation context, which share the original's variables, `graph.rs`, `upper_bound.rs`) | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/lib.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index d6fae1700..3f9b6aac8 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -76,7 +76,7 @@ same pipeline behind the `OptimizationPass` trait. | Pass 2 steps: rollup from the finest common group (shared producers across group keys), top-k reuse, readout requirements, dead-candidate elimination, chain groups (tumbling windows) and the logical edges | `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates/src/`): `lib.rs` (the steps in order), `rules/` (`candidates.rs`, `rollup/`, `topk.rs`, `chains.rs`), `readout_requirements.rs`, `dead_candidates.rs`, `edges.rs` and `evaluation_grids.rs`; the groups each query may read only exactly come from `asap_accuracy::ErrorMetricsOfMemo` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | none today (the standalone enumeration was deleted) | | What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `summary_type/capability.rs` | -| Stage 2: the timings and lifetimes offered per logical candidate and their prune rules, the per-group offers (ingestion time, across evaluations), the physical edges, and the alternatives no plan can use | `asap-stage2-physical-optimization`: `candidate.rs`, `group.rs`, `edge.rs`, `pruning.rs`; the admissible-edge indexes in `asap-types`, `stages/physical/memo_candidates/mod.rs` | +| Stage 2: the timings and lifetimes offered per logical candidate and their prune rules, the per-group offers (ingestion time, across evaluations), the physical edges, and the alternatives no plan can use | `asap-stage2-physical-optimization`: `candidate.rs`, `group.rs`, `edge.rs`, `pruning.rs`; the admissible-edge indexes, the times a stored result is kept at (`stored_times`), which contexts a candidate delivers on (`delivers_on`) and the candidates as the time walks see them (`TimeIndexedPlan`) in `asap-types`, `stages/physical/memo_candidates/mod.rs` | | Execution timing and data-state edge rules; each physical candidate's `PhysicalProperties` and `Lifetime`, and the physical edge check (`producer_satisfies_physical_requirements`) | `asap-types`, `ir/properties/execution_data_state/mod.rs`; `stages/physical/properties/mod.rs` (`PhysicalProperties`, `Lifetime`) and `stages/physical/candidate.rs` (`PhysicalCandidate`, the edge check) | | Stage 3: the execution contexts and the executions and edges a plan may select (`selectable.rs`), each execution's and each edge's cost and each retaining candidate's $\pi_v$ (`cost.rs`), how far back each edge reads its producer (`retention.rs`), the selection MILP over execution contexts with $h_v$ and (R), and with each query's exact-only reads $a_{q,e}$ and (X) (`milp/`, `exact_only.rs`), the plan it builds (`plan.rs`) and its report (`report.rs`); resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs, how far back time-indexed nodes are read | `asap-stage3-plan-selection`; `asap-cost`, `model.rs` (`CostModel`, `CostInputs`), `resources/mod.rs` (`ResourceUsage`), `measured.rs` and `plan/read_back.rs`; the time-window walks in `asap-types`, `plan/time_window.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `node/analytical.rs` and `node/operator_statistics.rs` | From 3417d63be5fae72ba38cb1b519a63d361533a8cf Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:45:10 +0000 Subject: [PATCH 21/23] fix(accuracy,stage3): a count's input is not exact-only; one retained output; targets in context copies (#1207 PR 7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit avg by (job)(quantile_over_time(0.99, data[300s])) at relative 0.05 over non-negative data had no plan without raw data: Pass 2 listed every DDSketch group as exact-only for it, and Stage 3 now enforces the list (X). The avg's exact-accumulator path reads count(quantile): a Project above the count passes no metric, so the count's group got none, and the walk and its closure marked the quantile and every sketch below it exact-only. A count's value does not depend on its input's error (error factor 0 in every metric), so a consumer now passes every metric in which an input's error does not reach its output, and the exact-only closure does not cross such an input. The DDSketch plan is valid: #1302 made a relative bound on avg hold for values of one sign, and data is declared non-negative. A store always keeps a stored result's newest output, and plan_cost charges at least one retained output, but Stage 3 let a stored result read only at its newest output retain 0 (an ingestion-time chain read by a stored ingestion-time TimeShift: 0.007/s apart). Each candidate with a retention cost now retains at least its output interval (ExecutionCosts::newest_output_retention_ms): the linear retention cost is taken at no less than one output, (R)'s least need, the bound and the reported retention are floored at it. Context copies also compare each query's accuracy target (metric, ε, δ, scope), so the accuracy constraint (K) holds alike in both copies: the same queries at rank 0.01 every 10 s and rank 0.05 every 20 s are not copies. --- crates/accuracy/src/metric_rule/mod.rs | 69 ++++++++++--- crates/accuracy/src/metric_rule/tests.rs | 45 +++++++++ crates/stage3-plan-selection/src/cost.rs | 34 ++++++- .../src/milp/equivalent_contexts.rs | 26 ++++- .../src/milp/model/tests.rs | 97 ++++++++++++++++++- .../src/milp/retention.rs | 9 +- crates/stage3-plan-selection/src/retention.rs | 19 +++- 7 files changed, 271 insertions(+), 28 deletions(-) diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index 51dd3e7e9..19018dd8f 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -126,6 +126,28 @@ impl ErrorMetricsOfMemo { .collect() } + /// The metrics in which no error of `consumer`'s input `input` reaches + /// its output (error factor 0): the input may be off in them whatever + /// its output allows. + /// + /// Example: `count` (an exact `Count` accumulator) outputs the number of + /// values it reads, whatever their error: every metric. + fn error_metrics_ignored_on_input( + &self, + consumer: CandidateForMemoGroupId, + input: usize, + ) -> Vec { + AccuracyMetric::ALL + .into_iter() + .filter(|&metric| { + self.error_contract(consumer, metric) + .is_some_and(|contract| { + contract.error_factor_per_input.get(input) == Some(&0.0) + }) + }) + .collect() + } + /// The metrics `consumer` allows an input's error in, the metric rule /// on its input edges: those its group's output allows that its error /// contract passes to its inputs (a readout's contract passes every @@ -144,13 +166,16 @@ impl ErrorMetricsOfMemo { /// down the live candidates (not in `dead`) from each root with a /// target, which allows its target's metrics /// ([`query_roots_with_target`]; an exact target none), a candidate passes to its inputs the - /// metrics its error contract passes; a group keeps, per query, the + /// metrics its error contract passes, and every metric in which an + /// input's error does not reach its output (factor 0: a `count`'s input + /// is never read exactly for its sake); a group keeps, per query, the /// metrics every path to it passes. A group is exact-only for a query /// when no metric reaches it (one path through a consumer without a /// contract, a comparison or a sum, suffices) or when it holds a sketch /// readout whose error metric is not among them (`quantile_over_time` /// at a relative target reads no KLL readout, whose error is a rank - /// one); and so is every group below one. + /// one); and so is every group below one, except through an input whose + /// error reaches its consumer's output in no metric. /// `exact_only_readout_groups_of_query` adds, per query, the readout /// groups the readout requirements list (a readout whose level another /// query gave, for a query whose failure probability has no per-value @@ -213,31 +238,43 @@ impl ErrorMetricsOfMemo { } } let passed = self.error_metrics_passed_to_inputs(consumer, &AccuracyMetric::ALL); - for &child in memo.candidate(consumer).expression.operator.children() { + for (input, &child) in memo + .candidate(consumer) + .expression + .operator + .children() + .into_iter() + .enumerate() + { + let ignored = self.error_metrics_ignored_on_input(consumer, input); for (&query, metrics) in &allowed_here { - let passed_here = metrics - .iter() - .copied() - .filter(|metric| passed.contains(metric)) + let passed_here = AccuracyMetric::ALL + .into_iter() + .filter(|metric| { + ignored.contains(metric) + || (metrics.contains(metric) && passed.contains(metric)) + }) .collect(); keep_common(&mut allowed, child, query, passed_here); } } } } - // Every group below an exact-only group is read exactly too. + // Every group below an exact-only group is read exactly too, except + // through an input whose error reaches the output in no metric (a + // `count`'s). for &group in order.iter().rev() { for groups in &mut exact_only { if groups.contains(&group) { for consumer in live_candidates(group) { - groups.extend( - memo.candidate(consumer) - .expression - .operator - .children() - .into_iter() - .copied(), - ); + let children = memo.candidate(consumer).expression.operator.children(); + groups.extend(children.into_iter().enumerate().filter_map( + |(input, &child)| { + (self.error_metrics_ignored_on_input(consumer, input).len() + < METRIC_COUNT) + .then_some(child) + }, + )); } } } diff --git a/crates/accuracy/src/metric_rule/tests.rs b/crates/accuracy/src/metric_rule/tests.rs index b76d523bb..f8e2ac447 100644 --- a/crates/accuracy/src/metric_rule/tests.rs +++ b/crates/accuracy/src/metric_rule/tests.rs @@ -332,3 +332,48 @@ fn a_scalar_roots_operands_are_roots() { check(scalar_of, vec![AccuracyMetric::Relative], true); check(doubled, vec![], false); } + +/// `count(quantile_over_time(0.99, m[5m]))` at an exact target: the +/// count of the quantiles does not depend on their values (its error +/// factor on its input is 0), so the quantile, its readouts and their +/// states are not exact-only for the query: the count's own group is, +/// and the raw window and scan below are (read only by the exact +/// quantile and the sketch builds, which pass no error). A +/// `count` beside a `sum` in `avg = sum / count` is such a path: it no +/// longer makes the quantile exact-only for a relative `avg`. +#[test] +fn a_count_reads_its_input_in_any_metric() { + let mut memo = Memo::new(&DataWorkload::default()); + let p99 = quantile_readouts(&mut memo, 0.99); + let count = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99.result, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Count], + output_names: vec!["count".into()], + filters: vec![None], + having: None, + }), + ); + query(&mut memo, count, Some(AccuracyTarget::exact())); + let order = memo.topological_order().expect("acyclic"); + let exact_only = error_metrics(&memo).exact_only_groups_of_query( + &memo, + &order, + &BTreeSet::new(), + &BTreeMap::new(), + ); + let [exact_only] = exact_only.as_slice() else { + panic!("one query: {exact_only:?}"); + }; + assert!(exact_only.contains(&count)); + let mut quantile_groups = vec![p99.result]; + for readout in &p99.by_algorithm { + quantile_groups.extend([readout.state, readout.readout.group]); + } + for group in quantile_groups { + assert!(!exact_only.contains(&group), "{group:?} in {exact_only:?}"); + } + assert_eq!(exact_only.len(), 3, "{exact_only:?}"); +} diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index 3239be54b..a4489cb32 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -99,8 +99,16 @@ pub(crate) struct ExecutionCosts { /// `None` when its producer retains no rows or the edge reads it /// unbounded. pub(crate) retention_need_ms_per_edge: Vec>, + /// Per candidate with a retention cost, its output interval Δ in ms + /// (`asap_cost::output_interval_ms`): a store always keeps a stored + /// result's newest output whatever its retention, so a selected one + /// retains at least Δ, as `plan_cost` charges at least one retained + /// output. Example: an ingestion-time chain of 60 s windows read only + /// at its newest window retains 60 s, not 0. + pub(crate) newest_output_retention_ms: HashMap, /// The least need, in ms, over each execution's costed reads, by - /// position in `SelectableExecutions::executions` + /// position in `SelectableExecutions::executions`, at least + /// [`Self::newest_output_retention_ms`] /// ([`least_need_ms_per_execution`]). pub(crate) least_need_ms_per_execution: Vec>, /// The source of each query-time `Scan`: its retention is raw data on @@ -346,9 +354,13 @@ pub(crate) fn execution_costs( // whole own cost when it is materialized, its raw data on disk for a // query-time scan (each of its runs reads what it reads at that // retention, ρ_κ c_{κ,v}). + // The retention is at least Δ: the newest output is always kept, so a + // retention under Δ costs what Δ does (`plan_cost` charges at least + // one retained output). let mut retention_cost_per_retained_second = HashMap::new(); + let mut newest_output_retention_ms = HashMap::new(); for (&v, &most_read_ms) in &most_read_ms { - let Some(retention_ms) = most_read_ms else { + let Some(most_read_ms) = most_read_ms else { continue; }; let window_ms = match nodes[v.0].output_state.timing { @@ -356,6 +368,7 @@ pub(crate) fn execution_costs( ExecutionTiming::QueryTime => None, }; let delta_ms = output_interval_ms(&nodes[v.0].payload, window_ms); + let retention_ms = most_read_ms.max(delta_ms); let longer = node_retaining(v, Some(retention_ms + delta_ms)); let delta_seconds = ms_to_seconds(delta_ms); let per_retained_second = match physical.physical_candidates[v.0] @@ -380,6 +393,7 @@ pub(crate) fn execution_costs( }; if let Some(per_retained_second) = per_retained_second { retention_cost_per_retained_second.insert(v, per_retained_second.max(0.0)); + newest_output_retention_ms.insert(v, delta_ms); } } // Each query-time scan's source and, without $h_v$, the raw data it @@ -422,7 +436,11 @@ pub(crate) fn execution_costs( .physical_properties .is_across_evaluations(), ) { - let retention_seconds = ms_to_seconds(most_read_ms[&v].unwrap_or(0)); + let retention_seconds = ms_to_seconds( + most_read_ms[&v] + .unwrap_or(0) + .max(newest_output_retention_ms[&v]), + ); own + raw_data - per_retained_second * retention_seconds } else { own + raw_data @@ -466,13 +484,19 @@ pub(crate) fn execution_costs( Some(with_u_own + with_u_raw_data - own - raw_data + scan_read) }) .collect(); - let least_need_ms_per_execution = - least_need_ms_per_execution(selectable, &per_edge, &retention_need_ms_per_edge, entries); + let least_need_ms_per_execution = least_need_ms_per_execution( + selectable, + &per_edge, + &retention_need_ms_per_edge, + &newest_output_retention_ms, + entries, + ); ExecutionCosts { per_execution, per_edge, retention_cost_per_retained_second, retention_need_ms_per_edge, + newest_output_retention_ms, least_need_ms_per_execution, raw_data_source, unbounded_raw_data_on_disk_cost_per_second, diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs index 43c558e69..1808e3253 100644 --- a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; +use asap_types::accuracy::AccuracyTarget; use asap_types::stages::{ ExecutionContext, ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId, }; @@ -63,11 +64,14 @@ struct ExecutionKey { } /// A query's part of a [`ContextKey`]: the executions that may answer each -/// of its outputs, its exact-only groups and its range-evaluation span. +/// of its outputs, its exact-only groups, its range-evaluation span and +/// its accuracy target (by index among the workload's distinct targets, +/// [`AccuracyTarget`] holding floats: metric, ε, δ and scope all equal). type QuerySignature<'a> = ( Vec>, Option<&'a BTreeSet>, u64, + usize, ); /// Which executions, edges and queries share variables @@ -77,7 +81,8 @@ type QuerySignature<'a> = ( /// they have the same [`ContextKey`] (the same candidates, kept alike, the /// same memo edges from producers that correspond, in the context or the /// same execution outside every evaluation context, and queries that pair -/// up with the same output executions, exact-only groups and span) and +/// up with the same output executions, exact-only groups, span and +/// accuracy target, so the accuracy constraint holds alike in both) and /// every cost in one is the same multiple $\alpha$ of the other's, to /// [`COST_RELATIVE_TOLERANCE`]. Each query belongs to its own evaluation /// context (`SelectableExecutions::evaluation_context`), wherever its @@ -143,6 +148,22 @@ pub(crate) fn copies_in_equivalent_contexts( } } }; + // Each query's accuracy target, as an index among the distinct ones. + let mut targets: Vec<&AccuracyTarget> = Vec::new(); + let target_of_query: Vec = inputs + .entries + .iter() + .map(|entry| { + let target = &entry.requirements.accuracy; + targets + .iter() + .position(|&known| known == target) + .unwrap_or_else(|| { + targets.push(target); + targets.len() - 1 + }) + }) + .collect(); let signature = |context: ExecutionContextId, query: QueryId| -> Option> { let outputs = outputs_of[&query] .iter() @@ -161,6 +182,7 @@ pub(crate) fn copies_in_equivalent_contexts( .exact_only_groups_of_query .get(query.0), inputs.entries[query.0].range_span_ms(), + target_of_query[query.0], )) }; let key_of = |context: ExecutionContextId| -> Option> { diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 050ced554..d84939e18 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeSet, HashMap}; +use asap_types::accuracy::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::operator::Reduction; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::ColumnRef; @@ -190,6 +191,7 @@ fn selection_inputs( }) .collect(), retention_cost_per_retained_second, + newest_output_retention_ms: HashMap::new(), least_need_ms_per_execution: Vec::new(), raw_data_source: HashMap::new(), unbounded_raw_data_on_disk_cost_per_second: HashMap::new(), @@ -198,6 +200,7 @@ fn selection_inputs( &selectable, &costs.per_edge, &costs.retention_need_ms_per_edge, + &costs.newest_output_retention_ms, entries, ); (selectable, costs) @@ -435,6 +438,75 @@ fn a_materialized_chain_runs_once_and_retains_what_its_consumers_read_back() { assert_eq!(chain_node.retention_ms, Some(300_000)); } +/// A chain of 10 s windows materialized across evaluations, read every +/// 10 s at its newest output only (no range above it): no read needs a +/// retention, but a store always keeps the newest output, so the chain +/// retains one window, 10 s, and pays for it (1 + 1 + 1e-3 · 10 = 2.01), +/// as `plan_cost` charges one retained output. Without the newest output +/// it would retain 0 and pay 2. +#[test] +fn a_stored_result_read_at_its_newest_output_retains_one_output() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let chain = group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: m, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 10_000, + }), + ); + query(&mut memo, chain, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(chain), ACROSS_EVALUATIONS), + ], + ); + let entries = every_10s(1); + let solved_with = |newest_output_ms: Option| { + let (selectable, mut costs) = + selection_inputs(&physical, &entries, &[1.0, 1.0], &[], &[(1, 1e-3)]); + if let Some(newest_output_ms) = newest_output_ms { + costs + .newest_output_retention_ms + .insert(PhysicalCandidateId(1), newest_output_ms); + costs.least_need_ms_per_execution = least_need_ms_per_execution( + &selectable, + &costs.per_edge, + &costs.retention_need_ms_per_edge, + &costs.newest_output_retention_ms, + &entries, + ); + } + solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap() + }; + let solved = solved_with(Some(10_000)); + assert_eq!( + solved.retention_ms.get(&PhysicalCandidateId(1)), + Some(&10_000) + ); + assert!((solved.cost_per_second_without_request_overhead - 2.01).abs() < 1e-9); + let without_newest_output = solved_with(None); + assert_eq!( + without_newest_output + .retention_ms + .get(&PhysicalCandidateId(1)), + Some(&0) + ); + assert!((without_newest_output.cost_per_second_without_request_overhead - 2.0).abs() < 1e-9); +} + /// Ga = `m offset 1s` and Gb = `m offset 2s`, each also computed from the /// other (Ga② = Gb shifted by −1 s, Gb② = Ga shifted by 1 s). Reading each /// other costs 1 and each own expression 10; without (E) both would read @@ -901,7 +973,9 @@ fn a_none_execution_its_one_evaluation_twin_dominates_is_left_out() { /// query share their variables, and the optimum reads the scan's /// `OneEvaluation` candidate in both (1.5 + 0 against 1 + 2), costing /// 2.5 + 1.25. With one cost of the 20 s context not half, they are not -/// copies. +/// copies; nor are they when the queries' accuracy targets differ (rank +/// 0.01 every 10 s, rank 0.05 every 20 s), which the accuracy constraint +/// (K) reads per query. #[test] fn a_context_equivalent_to_another_shares_its_variables() { let mut memo = empty_memo(); @@ -982,6 +1056,27 @@ fn a_context_equivalent_to_another_shares_its_variables() { let first_of_20s = (0..executions.len()) .find(|&index| every_20s(index)) .unwrap(); + let rank = + |max_error| AccuracyTarget::max_error(max_error).accuracy_metric(AccuracyMetric::Rank); + let mut differing_targets = entries.clone(); + differing_targets[0].requirements.accuracy = rank(0.01); + differing_targets[1].requirements.accuracy = rank(0.05); + let shared = copies_in_equivalent_contexts( + &SelectionInputs { + physical: &physical, + entries: &differing_targets, + selectable: &selectable, + costs: &costs, + }, + &kept, + ); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(1)]); + assert!(shared + .original_of_execution + .iter() + .enumerate() + .all(|(index, &original)| original == index)); + costs.per_execution[first_of_20s] = Ok(10.0); assert!(copies(&costs) .original_of_execution diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs index 6f53a03ce..367126268 100644 --- a/crates/stage3-plan-selection/src/milp/retention.rs +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -96,7 +96,8 @@ pub(crate) fn retention_constraints( /// The least $h_v$ (R) allows for a selection, in ms, of each candidate /// with a retention cost that `edges` (selected, by position in /// `SelectableExecutions::edges`) or `outputs` (the query outputs and the -/// executions answering them, by position) read: what the MILP's optimum +/// executions answering them, by position) read, at least its newest +/// output (`ExecutionCosts::newest_output_retention_ms`): what the MILP's optimum /// retains when $\pi_v > 0$, and what the plan retains. pub(crate) fn least_retention_ms( inputs: &SelectionInputs<'_>, @@ -122,5 +123,11 @@ pub(crate) fn least_retention_ms( *most = (*most).max(need_ms); } } + // At least the newest output, which a store always keeps. + for (v, most) in &mut retention_ms { + if let Some(&newest_ms) = inputs.costs.newest_output_retention_ms.get(v) { + *most = (*most).max(newest_ms); + } + } retention_ms } diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 0ace2bb4c..253181b94 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -8,6 +8,8 @@ //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 5, "Cost and retention"). +use std::collections::HashMap; + use asap_types::ir::Operator; use asap_types::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; use asap_types::stages::{ @@ -56,13 +58,16 @@ pub(crate) fn edge_scan_read_ms( /// The least need, in ms, over each execution's costed reads /// (`SelectableExecutions::reads_back`: an edge out of it `per_edge` costs, or a query output /// it may answer), by position in [`SelectableExecutions::executions`]; -/// `None` without a bounded one. (F) gives a selected execution a use and -/// (R) then retains at least that: the greedy bound charges it, and the -/// MILP has $h_v \ge$ it $\cdot x$. +/// `None` without a bounded one; at least its candidate's +/// `newest_output_retention_ms` (Δ, the newest output a store always +/// keeps). (F) gives a selected execution a use and (R) then retains at +/// least that: the greedy bound charges it, and the MILP has $h_v \ge$ it +/// $\cdot x$. pub(crate) fn least_need_ms_per_execution( selectable: &SelectableExecutions, per_edge: &[Option], need_ms_per_edge: &[Option], + newest_output_retention_ms: &HashMap, entries: &[QueryWorkloadEntry], ) -> Vec> { let mut least_need_ms = vec![None; selectable.executions().len()]; @@ -76,6 +81,14 @@ pub(crate) fn least_need_ms_per_execution( *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); } } + for (least, execution) in least_need_ms.iter_mut().zip(selectable.executions()) { + if let (Some(least), Some(&newest_ms)) = ( + least.as_mut(), + newest_output_retention_ms.get(&execution.candidate), + ) { + *least = (*least).max(newest_ms); + } + } least_need_ms } From 44a223d9374e54140952e8b60186a63d9ff975f6 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:06:34 +0000 Subject: [PATCH 22/23] fix(stage3,accuracy,cost,types): round-5 review fixes: context copies to the context's largest cost, one metric rule per input, plan children order (#1207 PR 7) - Context copies compare costs to COST_RELATIVE_TOLERANCE of the copy's largest cost, alpha from that pair: a small edge cost, a difference of large costs, keeps their rounding (RollupHeavy-8users flows: -2.36e-15 and -4.71e-16, ratio 0.200000000026), which stopped copies sharing. - One metric rule per input: ErrorContract::input_error_reaches_output and ErrorMetricsOfMemo::error_metrics_allowed_on_input, used by the exact-only walk and closure, the edges and the readout caps; a count's input edge from a readout holds in any metric. - Newest-output floor applied once: most_read_ms floored where built, ExecutionCosts::retention_ms_for_need for edge and output needs. - Operator::map_children applies f in children() order, so a plan's Project over q with scalar(count(q)) keeps q as its child; plan_cost takes a node's producers in its payload's input order. - Layering: From for EvaluationGrid in plan; ExecutionContext::is_outside_evaluations; QuerySignature with named fields; QueryWorkloadEntry::evaluation_grid; exact-only field docs. - Abstraction: key_of returns queries in signature order; tolerance comments; range_span_ms for @ start() spans. --- crates/accuracy/src/error_contract/mod.rs | 15 ++ crates/accuracy/src/logical_edge/mod.rs | 4 +- crates/accuracy/src/metric_rule/mod.rs | 89 ++++----- crates/accuracy/src/metric_rule/tests.rs | 40 +++- crates/accuracy/src/readout_caps/mod.rs | 4 +- crates/cost/src/node/usage/mod.rs | 5 +- crates/cost/src/plan/cost.rs | 9 + crates/cost/src/plan/execution_demand/mod.rs | 6 +- .../tests/stage3_objective.rs | 30 +++ crates/stage3-plan-selection/src/cost.rs | 113 ++++++----- .../src/milp/dominance.rs | 1 + .../src/milp/equivalent_contexts.rs | 187 ++++++++++-------- .../src/milp/model/tests.rs | 79 ++++++-- .../src/milp/retention.rs | 13 +- .../src/milp/upper_bound.rs | 10 +- crates/stage3-plan-selection/src/retention.rs | 39 ++-- .../stage3-plan-selection/src/selectable.rs | 15 +- crates/types/src/ir/operator/node.rs | 1 + crates/types/src/ir/operator/non_asap.rs | 48 ++--- .../types/src/plan/materialized_view/mod.rs | 14 ++ .../src/stages/logical/memo_candidates.rs | 23 +-- .../src/stages/physical/execution_context.rs | 14 ++ crates/types/src/workload/query.rs | 12 ++ 23 files changed, 469 insertions(+), 302 deletions(-) diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs index 0ddbd7b7a..2ad2a9a19 100644 --- a/crates/accuracy/src/error_contract/mod.rs +++ b/crates/accuracy/src/error_contract/mod.rs @@ -336,5 +336,20 @@ pub struct ErrorContract { pub linear_bound_holds_up_to_per_input: Vec>, } +impl ErrorContract { + /// Whether any of input `input`'s error reaches the output + /// ($\beta_{c,i} > 0$). Where it does not, the input may be off in the + /// contract's metric whatever the output allows. + /// + /// Example: `count` outputs how many values it reads, whatever their + /// error ($\beta = 0$): false; `sum` passes its input's relative error + /// on ($\beta = 1$): true. + pub fn input_error_reaches_output(&self, input: usize) -> bool { + self.error_factor_per_input + .get(input) + .is_none_or(|&factor| factor > 0.0) + } +} + #[cfg(test)] mod tests; diff --git a/crates/accuracy/src/logical_edge/mod.rs b/crates/accuracy/src/logical_edge/mod.rs index c99686394..56542b462 100644 --- a/crates/accuracy/src/logical_edge/mod.rs +++ b/crates/accuracy/src/logical_edge/mod.rs @@ -73,7 +73,7 @@ impl ErrorMetricsOfMemo { let Some(producer_metric) = self.readout_error_metric(producer) else { return Ok(()); }; - return match self.error_metrics_allowed_on_inputs(consumer) { + return match self.error_metrics_allowed_on_input_edge(consumer, input) { Some(allowed) => metric_rule(producer_metric, &allowed), None => Ok(()), }; @@ -100,7 +100,7 @@ impl ErrorMetricsOfMemo { let failure_probability = required.map_or(1.0, |required| required.failure_probability); let measured = evidence.measured_guarantee(summary_type, parameters, statistic, failure_probability); - let allowed = self.error_metrics_allowed_on_inputs(consumer); + let allowed = self.error_metrics_allowed_on_input_edge(consumer, input); // The theoretical guarantee before its failure-probability check, // computed once for both checks below. let theoretical = (measured.is_none() && (allowed.is_some() || required.is_some())) diff --git a/crates/accuracy/src/metric_rule/mod.rs b/crates/accuracy/src/metric_rule/mod.rs index 19018dd8f..9d15c30be 100644 --- a/crates/accuracy/src/metric_rule/mod.rs +++ b/crates/accuracy/src/metric_rule/mod.rs @@ -46,7 +46,12 @@ impl ErrorMetricsOfMemo { /// input those its output allows that its error contract passes to it; /// a group the union over its consumers. These are the metrics of the /// readout caps' walk (`readout_caps`), which also reaches a group - /// through inputs the output does not depend on. + /// through inputs the output does not depend on. A metric only such an + /// input would allow is not added here: a readout's edge into such a + /// consumer is admitted by [`Self::error_metrics_allowed_on_input`], + /// and widening a whole group for it would admit every alternative + /// readout of the group (`avg` of a relative quantile: 62 → 455 + /// MILP variables, the same plan). pub fn of(memo: &Memo, order: &[MemoGroupId]) -> Self { let error_contract: Vec> = memo .groups() @@ -111,71 +116,63 @@ impl ErrorMetricsOfMemo { self.readout_error_metric[candidate.group.0 as usize][candidate.index_in_group] } - /// The metrics among `metrics` that `consumer`'s error contract passes - /// to its inputs: those an input of it may be off in when its output - /// may be off in `metrics`. - fn error_metrics_passed_to_inputs( - &self, - consumer: CandidateForMemoGroupId, - metrics: &[AccuracyMetric], - ) -> Vec { - metrics - .iter() - .copied() - .filter(|&metric| self.error_contract(consumer, metric).is_some()) - .collect() - } - - /// The metrics in which no error of `consumer`'s input `input` reaches - /// its output (error factor 0): the input may be off in them whatever - /// its output allows. + /// The metrics `consumer`'s input `input` may be off in when its + /// output may be off in `output_allowed`, the metric rule: those of + /// `output_allowed` its error contract passes to its inputs, and every + /// metric in which the input's error does not reach the output + /// ([`ErrorContract::input_error_reaches_output`]). In + /// `AccuracyMetric::ALL` order. /// - /// Example: `count` (an exact `Count` accumulator) outputs the number of - /// values it reads, whatever their error: every metric. - fn error_metrics_ignored_on_input( + /// Example: a readout's contract passes every metric, so a readout + /// allows its output's; `count` (error factor 0) allows every metric, + /// even under an exact target; a sum of values of unknown sign under a + /// relative target allows none. + pub(crate) fn error_metrics_allowed_on_input( &self, consumer: CandidateForMemoGroupId, input: usize, + output_allowed: &[AccuracyMetric], ) -> Vec { AccuracyMetric::ALL .into_iter() .filter(|&metric| { self.error_contract(consumer, metric) .is_some_and(|contract| { - contract.error_factor_per_input.get(input) == Some(&0.0) + output_allowed.contains(&metric) + || !contract.input_error_reaches_output(input) }) }) .collect() } - /// The metrics `consumer` allows an input's error in, the metric rule - /// on its input edges: those its group's output allows that its error - /// contract passes to its inputs (a readout's contract passes every - /// metric, so a readout allows its output's). `None` where no query - /// with a target reaches the consumer. - pub(crate) fn error_metrics_allowed_on_inputs( + /// [`Self::error_metrics_allowed_on_input`] at what `consumer`'s group's + /// output allows, the metric rule on its input edges; `None` where no + /// query with a target reaches the consumer. + pub(crate) fn error_metrics_allowed_on_input_edge( &self, consumer: CandidateForMemoGroupId, + input: usize, ) -> Option> { let allowed = self.error_metrics_allowed_at_output[consumer.group.0 as usize].as_ref()?; - Some(self.error_metrics_passed_to_inputs(consumer, allowed)) + Some(self.error_metrics_allowed_on_input(consumer, input, allowed)) } /// Per query: the groups below its root it may read only exactly, the /// metric rule per query where the edges apply it per group. Walking /// down the live candidates (not in `dead`) from each root with a /// target, which allows its target's metrics - /// ([`query_roots_with_target`]; an exact target none), a candidate passes to its inputs the - /// metrics its error contract passes, and every metric in which an - /// input's error does not reach its output (factor 0: a `count`'s input - /// is never read exactly for its sake); a group keeps, per query, the + /// ([`query_roots_with_target`]; an exact target none), a candidate + /// passes to each input [`Self::error_metrics_allowed_on_input`] (a + /// `count`'s input is never read exactly for its sake); a group keeps, + /// per query, the /// metrics every path to it passes. A group is exact-only for a query /// when no metric reaches it (one path through a consumer without a /// contract, a comparison or a sum, suffices) or when it holds a sketch /// readout whose error metric is not among them (`quantile_over_time` /// at a relative target reads no KLL readout, whose error is a rank - /// one); and so is every group below one, except through an input whose - /// error reaches its consumer's output in no metric. + /// one); and so is every group below one, except through an input + /// that may be off in some metric under an output allowing none (a + /// `count`'s). /// `exact_only_readout_groups_of_query` adds, per query, the readout /// groups the readout requirements list (a readout whose level another /// query gave, for a query whose failure probability has no per-value @@ -237,7 +234,6 @@ impl ErrorMetricsOfMemo { } } } - let passed = self.error_metrics_passed_to_inputs(consumer, &AccuracyMetric::ALL); for (input, &child) in memo .candidate(consumer) .expression @@ -246,23 +242,16 @@ impl ErrorMetricsOfMemo { .into_iter() .enumerate() { - let ignored = self.error_metrics_ignored_on_input(consumer, input); for (&query, metrics) in &allowed_here { - let passed_here = AccuracyMetric::ALL - .into_iter() - .filter(|metric| { - ignored.contains(metric) - || (metrics.contains(metric) && passed.contains(metric)) - }) - .collect(); - keep_common(&mut allowed, child, query, passed_here); + let passed = self.error_metrics_allowed_on_input(consumer, input, metrics); + keep_common(&mut allowed, child, query, passed); } } } } // Every group below an exact-only group is read exactly too, except - // through an input whose error reaches the output in no metric (a - // `count`'s). + // through an input that may be off in some metric under an output + // allowing none (a `count`'s). for &group in order.iter().rev() { for groups in &mut exact_only { if groups.contains(&group) { @@ -270,8 +259,8 @@ impl ErrorMetricsOfMemo { let children = memo.candidate(consumer).expression.operator.children(); groups.extend(children.into_iter().enumerate().filter_map( |(input, &child)| { - (self.error_metrics_ignored_on_input(consumer, input).len() - < METRIC_COUNT) + self.error_metrics_allowed_on_input(consumer, input, &[]) + .is_empty() .then_some(child) }, )); diff --git a/crates/accuracy/src/metric_rule/tests.rs b/crates/accuracy/src/metric_rule/tests.rs index f8e2ac447..a4c186fec 100644 --- a/crates/accuracy/src/metric_rule/tests.rs +++ b/crates/accuracy/src/metric_rule/tests.rs @@ -25,12 +25,13 @@ fn allowed_at_output(memo: &Memo, group: MemoGroupId) -> Option Option> { - error_metrics(memo).error_metrics_allowed_on_inputs(consumer) + error_metrics(memo).error_metrics_allowed_on_input_edge(consumer, input) } /// $\alpha^{\log}$ of the edge from `producer` into input `input` of @@ -93,7 +94,7 @@ fn a_ratio_reads_no_rank_error() { query(&mut memo, ratio, Some(target)); let relative = Some(vec![AccuracyMetric::Relative]); assert_eq!(allowed_at_output(&memo, p99), relative); - assert_eq!(allowed_on_inputs(&memo, own_candidate(ratio)), relative); + assert_eq!(allowed_on_input(&memo, own_candidate(ratio), 0), relative); assert!(matches!( edge(&memo, own_candidate(p99), own_candidate(ratio), 0), Err(RootPruneReason::MetricRule { @@ -143,7 +144,7 @@ fn the_metric_rule_cuts_the_readout_edges_and_keeps_the_exact_division() { let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::ABSOLUTE); query(&mut memo, avg, Some(target)); let division = own_candidate(ratio); - assert_eq!(allowed_on_inputs(&memo, division), Some(vec![])); + assert_eq!(allowed_on_input(&memo, division, 0), Some(vec![])); let result = edge(&memo, division, own_candidate(sum), 0); assert!(result.is_ok(), "{result:?}"); for input in 0..2 { @@ -195,7 +196,7 @@ fn a_key_only_filter_keeps_the_ddsketch_readout_admissible() { let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); query(&mut memo, filtered, Some(target)); assert_eq!( - allowed_on_inputs(&memo, own_candidate(filtered)), + allowed_on_input(&memo, own_candidate(filtered), 0), Some(vec![AccuracyMetric::Relative]) ); let result = readout_edge(&memo, &p99.by_algorithm[0], own_candidate(filtered), 0); @@ -377,3 +378,30 @@ fn a_count_reads_its_input_in_any_metric() { } assert_eq!(exact_only.len(), 3, "{exact_only:?}"); } + +/// A `count`'s output does not depend on its input's error (error factor +/// 0 in every metric), so the metric rule lets its input be off in any +/// metric, on its edges as in `exact_only_groups_of_query`. +/// `count(quantile_over_time(0.99, m[5m]))` with a relative target, the +/// count reading the KLL readout directly: the readout's rank error is not +/// one the output allows, but its edge into the `count` holds. +#[test] +fn a_counts_input_edge_from_a_readout_holds_in_a_metric_its_output_does_not_allow() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, _, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let count = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Count], + output_names: vec!["count".into()], + filters: vec![None], + having: None, + }), + ); + let relative = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, count, Some(relative)); + let result = edge(&memo, own_candidate(p99), own_candidate(count), 0); + assert!(result.is_ok(), "{result:?}"); +} diff --git a/crates/accuracy/src/readout_caps/mod.rs b/crates/accuracy/src/readout_caps/mod.rs index c0784392f..be3925b49 100644 --- a/crates/accuracy/src/readout_caps/mod.rs +++ b/crates/accuracy/src/readout_caps/mod.rs @@ -265,10 +265,10 @@ pub(crate) fn caps( for (input, &child) in children.iter().enumerate() { let child_caps = caps[child.0 as usize].get_or_insert_with(HashMap::new); child_caps.entry(metric).or_default(); - let factor = contract.error_factor_per_input[input]; // An input the output does not depend on (a count's, // β = 0) gets no cap from it. - if factor > 0.0 { + if contract.input_error_reaches_output(input) { + let factor = contract.error_factor_per_input[input]; let holds_up_to = contract.linear_bound_holds_up_to_per_input[input] .unwrap_or(f64::INFINITY); for (&query, &cap) in caps_per_query { diff --git a/crates/cost/src/node/usage/mod.rs b/crates/cost/src/node/usage/mod.rs index 8a2dccba0..f347c1259 100644 --- a/crates/cost/src/node/usage/mod.rs +++ b/crates/cost/src/node/usage/mod.rs @@ -236,8 +236,9 @@ pub fn output_interval_ms(payload: &Operator, window_ms: Option) -> u /// Measure and cost one node of a physical DAG (one step of /// [`plan_cost`]; Stage 3 calls it per candidate): `producers` are its -/// producers in edge order, `rate` its `(multiplier, per_second)` -/// ([`computations`], [`executions_per_second`]), and `window_ms` the time +/// producers in its payload's input order (`Operator::children`), `rate` +/// its `(multiplier, per_second)` ([`computations`], +/// [`executions_per_second`]), and `window_ms` the time /// window one execution covers: an ingestion-time node's refresh window /// width (`RefreshWindow::computed_from`, the rule the installed plan /// refreshes it on), or a query-time `Scan`'s read, the diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 599b19dc8..9d9e5d1ec 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -105,10 +105,19 @@ pub fn plan_cost( // usage and cost ([`node_resource_usage_and_cost`]), children first. let multipliers = computations(dag, entries, settings.non_repeating_query_horizon_seconds) .map_err(|error| (first, error))?; + // Each node's producers in its payload's input order (its children, + // then the nodes its scalar expressions refer to), whatever order the + // DAG lists its edges in: its first producer is its child. Example: + // `q + scalar(count(q))` as a `Project` over `q` reads `q` first, even + // when the edge from the count is listed first. let mut producers_of: HashMap> = HashMap::new(); for e in &dag.edges { producers_of.entry(e.consumer).or_default().push(e.producer); } + for (id, producers) in &mut producers_of { + let inputs = nodes[id].payload.children(); + producers.sort_by_key(|producer| inputs.iter().position(|&input| input == producer)); + } let first_input_of = |id: PhysicalASAPNodeId| { let input = producers_of.get(&id)?.first()?; Some(&nodes[input].payload) diff --git a/crates/cost/src/plan/execution_demand/mod.rs b/crates/cost/src/plan/execution_demand/mod.rs index 2004b1a05..24d8efe55 100644 --- a/crates/cost/src/plan/execution_demand/mod.rs +++ b/crates/cost/src/plan/execution_demand/mod.rs @@ -395,10 +395,12 @@ pub fn context_executions_per_second<'e>( // Requests of one span need the input at one start time each; // different spans need it at different times of one request. Some(AtModifier::Start) => { - let mut by_span: BTreeMap, Vec<&QueryRecurrence>> = BTreeMap::new(); + // An instant query anchors `@ start()` where a span-0 range + // query does. + let mut by_span: BTreeMap> = BTreeMap::new(); for entry in queries { by_span - .entry(entry.range_evaluation.map(|range| range.span_ms)) + .entry(entry.range_span_ms()) .or_default() .push(&entry.recurrence); } diff --git a/crates/integration-tests/tests/stage3_objective.rs b/crates/integration-tests/tests/stage3_objective.rs index 1e48dd939..51100e33f 100644 --- a/crates/integration-tests/tests/stage3_objective.rs +++ b/crates/integration-tests/tests/stage3_objective.rs @@ -604,3 +604,33 @@ fn the_objective_is_the_plan_cost_when_two_queries_read_one_anchored_group() { selected_plan_objective_and_plan_cost_with(&w, models, &CostInputs::default()).1, ); } + +/// `q + scalar(count(q))` with `q` = `quantile_over_time(0.99, m[5m])`, +/// exact, raw data kept: the `+`'s `Project` reads `q` as its child and +/// the count through `scalar()`. The plan's `Project` keeps `q` as its +/// child (building it handed the count in as the child, as +/// `Operator::map_children` mapped the scalar reference first), and +/// `plan_cost` sizes it by its child, `q`'s 1,000 rows, as the MILP does, +/// not by the count's one row, whatever order the plan lists its edges in. +#[test] +fn the_objective_is_the_plan_cost_on_a_scalar_operand() { + let w = workload(&[( + "quantile_over_time(0.99, m[5m]) + scalar(count(quantile_over_time(0.99, m[5m])))", + 60_000, + "", + )]); + let settings = CostInputs::default(); + let (plan, costs) = selected_plan_objective_and_plan_cost(&w, &settings, true); + assert_equal("scalar operand", costs); + let entries: Vec<_> = w.query_workload.entries().collect(); + let data = w.data_workload.clone().unwrap(); + let mut edges_reversed = plan.clone(); + edges_reversed.edges.reverse(); + let cost_of = |plan| { + asap_cost::plan_cost(plan, &entries, &data, &settings) + .unwrap() + .0 + .cost_per_second + }; + assert_eq!(cost_of(&edges_reversed), cost_of(&plan)); +} diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index a4489cb32..a2fa2ebd0 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -99,17 +99,18 @@ pub(crate) struct ExecutionCosts { /// `None` when its producer retains no rows or the edge reads it /// unbounded. pub(crate) retention_need_ms_per_edge: Vec>, - /// Per candidate with a retention cost, its output interval Δ in ms - /// (`asap_cost::output_interval_ms`): a store always keeps a stored - /// result's newest output whatever its retention, so a selected one - /// retains at least Δ, as `plan_cost` charges at least one retained - /// output. Example: an ingestion-time chain of 60 s windows read only - /// at its newest window retains 60 s, not 0. + /// Per candidate that retains rows, its output interval Δ in ms + /// (`asap_cost::output_interval_ms`; a chain's window, the window an + /// ingestion-time view maintains, else a second): a store always keeps a + /// stored result's newest output whatever its retention, so a selected + /// one retains at least Δ, as `plan_cost` charges at least one retained + /// output ([`Self::retention_ms_for_need`]). Example: an ingestion-time + /// chain of 60 s windows read only at its newest window retains 60 s, + /// not 0. pub(crate) newest_output_retention_ms: HashMap, - /// The least need, in ms, over each execution's costed reads, by - /// position in `SelectableExecutions::executions`, at least - /// [`Self::newest_output_retention_ms`] - /// ([`least_need_ms_per_execution`]). + /// The least retention, in ms, over each execution's costed reads + /// ([`Self::retention_ms_for_need`] of each), by position in + /// `SelectableExecutions::executions` ([`least_need_ms_per_execution`]). pub(crate) least_need_ms_per_execution: Vec>, /// The source of each query-time `Scan`: its retention is raw data on /// disk, which a plan keeps once per source for the selected scan that @@ -121,6 +122,18 @@ pub(crate) struct ExecutionCosts { } impl ExecutionCosts { + /// What candidate `v` retains, in ms, for a read needing `need_ms`: at + /// least its newest output ([`Self::newest_output_retention_ms`]), + /// which a store always keeps. + /// + /// Example: a chain of 60 s windows read at its newest window (need 0) + /// retains 60,000 ms; read over the last 5 minutes, 300,000 ms. + pub(crate) fn retention_ms_for_need(&self, v: PhysicalCandidateId, need_ms: u64) -> u64 { + self.newest_output_retention_ms + .get(&v) + .map_or(need_ms, |&newest_ms| need_ms.max(newest_ms)) + } + /// $\pi_v h_v$ of candidate `v` retaining `retention_ms`, per second; 0 /// for a candidate with no retention cost. pub(crate) fn retention_cost_per_second( @@ -182,19 +195,31 @@ pub(crate) fn execution_costs( .or_insert_with(|| edge_retention_need_ms(physical, entries, edge)) }) .collect(); - // The most any consumer of each candidate that retains rows reads back - // ($h_v$ at its largest): `None` when one reads it unbounded. - let mut most_read_ms: HashMap> = (0..count) + // Δ of each candidate that retains rows, the newest output a store + // always keeps (`ExecutionCosts::newest_output_retention_ms`). + let newest_output_retention_ms: HashMap = (0..count) .filter(|&v| retains_rows[v]) - .map(|v| (PhysicalCandidateId(v), Some(0))) + .map(PhysicalCandidateId) + .map(|v| { + let node = node_retaining(v, None); + let window_ms = match node.output_state.timing { + ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), + ExecutionTiming::QueryTime => None, + }; + (v, output_interval_ms(&node.payload, window_ms)) + }) .collect(); - let mut read = |v: PhysicalCandidateId, need_ms: Option| { - if let Some(most) = most_read_ms.get_mut(&v) { + // The most any consumer of each candidate that retains rows reads back + // ($h_v$ at its largest), at least its newest output: `None` when one + // reads it unbounded. + let mut most_read_ms: HashMap> = newest_output_retention_ms + .iter() + .map(|(&v, &newest_ms)| (v, Some(newest_ms))) + .collect(); + for (index, _, need_ms) in selectable.reads_back(&retention_need_ms_per_edge, entries) { + if let Some(most) = most_read_ms.get_mut(&executions[index].candidate) { *most = most.zip(need_ms).map(|(most, need)| most.max(need)); } - }; - for (index, _, need_ms) in selectable.reads_back(&retention_need_ms_per_edge, entries) { - read(executions[index].candidate, need_ms); } // How far back a query-time `Scan` reads for the consumer of each edge // out of it, in one run (`None`: read whole), once per memo edge. @@ -347,29 +372,18 @@ pub(crate) fn execution_costs( .map(|(index, execution)| (execution.candidate, index)) .collect(); // π_v of a candidate retaining rows every consumer reads within a bound: - // what one more Δ ms of retention costs, per second retained (Δ the - // time between two of its outputs: a chain's window, the window an - // ingestion-time view maintains, else a second; a retention is a whole - // number of outputs), at its retention's largest; its - // whole own cost when it is materialized, its raw data on disk for a - // query-time scan (each of its runs reads what it reads at that - // retention, ρ_κ c_{κ,v}). - // The retention is at least Δ: the newest output is always kept, so a - // retention under Δ costs what Δ does (`plan_cost` charges at least - // one retained output). + // what one more Δ ms of retention (one more output; a retention is a + // whole number of outputs) costs, per second retained, at its + // retention's largest; its whole own cost when it is materialized, its + // raw data on disk for a query-time scan (each of its runs reads what + // it reads at that retention, ρ_κ c_{κ,v}). let mut retention_cost_per_retained_second = HashMap::new(); - let mut newest_output_retention_ms = HashMap::new(); for (&v, &most_read_ms) in &most_read_ms { let Some(most_read_ms) = most_read_ms else { continue; }; - let window_ms = match nodes[v.0].output_state.timing { - ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), - ExecutionTiming::QueryTime => None, - }; - let delta_ms = output_interval_ms(&nodes[v.0].payload, window_ms); - let retention_ms = most_read_ms.max(delta_ms); - let longer = node_retaining(v, Some(retention_ms + delta_ms)); + let delta_ms = newest_output_retention_ms[&v]; + let longer = node_retaining(v, Some(most_read_ms + delta_ms)); let delta_seconds = ms_to_seconds(delta_ms); let per_retained_second = match physical.physical_candidates[v.0] .physical_properties @@ -393,7 +407,6 @@ pub(crate) fn execution_costs( }; if let Some(per_retained_second) = per_retained_second { retention_cost_per_retained_second.insert(v, per_retained_second.max(0.0)); - newest_output_retention_ms.insert(v, delta_ms); } } // Each query-time scan's source and, without $h_v$, the raw data it @@ -436,12 +449,9 @@ pub(crate) fn execution_costs( .physical_properties .is_across_evaluations(), ) { - let retention_seconds = ms_to_seconds( - most_read_ms[&v] - .unwrap_or(0) - .max(newest_output_retention_ms[&v]), - ); - own + raw_data - per_retained_second * retention_seconds + let retention_ms = + most_read_ms[&v].expect("a candidate with π_v is read within a bound"); + own + raw_data - per_retained_second * ms_to_seconds(retention_ms) } else { own + raw_data }) @@ -484,23 +494,18 @@ pub(crate) fn execution_costs( Some(with_u_own + with_u_raw_data - own - raw_data + scan_read) }) .collect(); - let least_need_ms_per_execution = least_need_ms_per_execution( - selectable, - &per_edge, - &retention_need_ms_per_edge, - &newest_output_retention_ms, - entries, - ); - ExecutionCosts { + let mut costs = ExecutionCosts { per_execution, per_edge, retention_cost_per_retained_second, retention_need_ms_per_edge, newest_output_retention_ms, - least_need_ms_per_execution, + least_need_ms_per_execution: Vec::new(), raw_data_source, unbounded_raw_data_on_disk_cost_per_second, - } + }; + costs.least_need_ms_per_execution = least_need_ms_per_execution(selectable, &costs, entries); + costs } /// $\rho$ of `execution`'s context, how often it runs, from the queries it diff --git a/crates/stage3-plan-selection/src/milp/dominance.rs b/crates/stage3-plan-selection/src/milp/dominance.rs index 6dda54bf3..3287909fd 100644 --- a/crates/stage3-plan-selection/src/milp/dominance.rs +++ b/crates/stage3-plan-selection/src/milp/dominance.rs @@ -34,6 +34,7 @@ pub(crate) fn dominated_by_one_evaluation_twin(inputs: &SelectionInputs<'_>) -> let executions = selectable.executions(); let edges = selectable.edges(); let count = executions.len(); + // No more, to rounding (to 1e-24/s when `than` is 0). let no_more = |cost: f64, than: f64| cost <= than + COST_RELATIVE_TOLERANCE * than.abs().max(1e-12); let mut answers: Vec> = vec![BTreeSet::new(); count]; diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs index 1808e3253..2dd4dcc90 100644 --- a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -6,9 +6,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use asap_types::accuracy::AccuracyTarget; -use asap_types::stages::{ - ExecutionContext, ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId, -}; +use asap_types::stages::{ExecutionContextId, MemoGroupId, PhysicalCandidateId, PhysicalEdgeId}; use super::model::SelectionInputs; use super::COST_RELATIVE_TOLERANCE; @@ -63,16 +61,19 @@ struct ExecutionKey { inputs: Vec>, } -/// A query's part of a [`ContextKey`]: the executions that may answer each -/// of its outputs, its exact-only groups, its range-evaluation span and -/// its accuracy target (by index among the workload's distinct targets, -/// [`AccuracyTarget`] holding floats: metric, ε, δ and scope all equal). -type QuerySignature<'a> = ( - Vec>, - Option<&'a BTreeSet>, - u64, - usize, -); +/// A query's part of a [`ContextKey`]. +#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct QuerySignature<'a> { + /// Per output, the executions that may answer it. + answering_executions_per_output: Vec>, + /// The groups it may read only exactly. + exact_only_groups: Option<&'a BTreeSet>, + /// Its range-evaluation span (`QueryWorkloadEntry::range_span_ms`). + range_span_ms: u64, + /// Its accuracy target, by index among the workload's distinct targets + /// ([`AccuracyTarget`] holds floats: metric, ε, δ and scope all equal). + accuracy_target_index: usize, +} /// Which executions, edges and queries share variables /// ([`SharedVariables`]): a root evaluation context (no request anchor) @@ -84,10 +85,11 @@ type QuerySignature<'a> = ( /// up with the same output executions, exact-only groups, span and /// accuracy target, so the accuracy constraint holds alike in both) and /// every cost in one is the same multiple $\alpha$ of the other's, to -/// [`COST_RELATIVE_TOLERANCE`]. Each query belongs to its own evaluation -/// context (`SelectableExecutions::evaluation_context`), wherever its -/// answering executions run. Contexts are bucketed by key, so only the -/// cost check compares two contexts. +/// [`COST_RELATIVE_TOLERANCE`] of the copy's largest cost. Each query +/// belongs to its own evaluation context +/// (`SelectableExecutions::evaluation_context`), wherever its answering +/// executions run. Contexts are bucketed by key, so only the cost check +/// compares two contexts. /// /// Exact: from an optimal plan, give both contexts the selection of the /// one whose own cost $A$ or $B$ is lower, say $A$. The copied context @@ -95,7 +97,9 @@ type QuerySignature<'a> = ( /// $r_s$ (each the most of their readers' needs, equal in both copies) do /// not grow, and an execution outside them that only the other selection /// read is left out at no cost; with $B \le A$ symmetrically. So some -/// optimum selects alike in both, and sharing their variables keeps it. +/// optimum selects alike in both, and sharing their variables keeps it, +/// to the tolerance the costs are compared to (each copied cost within +/// 1e-12 of the context's largest cost of $\alpha$ times its original). /// /// Example: the same queries every 60 s and every 300 s (one user each): /// the 300 s context's executions cost 1/5 of the 60 s ones' and share @@ -128,8 +132,10 @@ pub(crate) fn copies_in_equivalent_contexts( for (query, answering) in selectable.answering() { outputs_of.entry(*query).or_default().push(answering); } + // Each evaluation context's queries; a query without an output has + // none. let mut queries_of: BTreeMap> = BTreeMap::new(); - for &query in outputs_of.keys() { + for query in (0..inputs.entries.len()).map(QueryId) { if let Some(context) = selectable.evaluation_context(query) { queries_of.entry(context).or_default().push(query); } @@ -138,14 +144,13 @@ pub(crate) fn copies_in_equivalent_contexts( // context (a request context), which no copy shares. let in_context = |context: ExecutionContextId, index: usize| { let execution_context = executions[index].context; - match selectable.context(execution_context) { - _ if execution_context == context => { - position_in_context[index].map(ExecutionInContext::Own) - } - ExecutionContext::Evaluation { .. } => None, - ExecutionContext::IngestionPipeline(_) | ExecutionContext::Materialized(_) => { - Some(ExecutionInContext::Shared(index)) - } + if execution_context == context { + position_in_context[index].map(ExecutionInContext::Own) + } else { + selectable + .context(execution_context) + .is_outside_evaluations() + .then_some(ExecutionInContext::Shared(index)) } }; // Each query's accuracy target, as an index among the distinct ones. @@ -165,8 +170,10 @@ pub(crate) fn copies_in_equivalent_contexts( }) .collect(); let signature = |context: ExecutionContextId, query: QueryId| -> Option> { - let outputs = outputs_of[&query] - .iter() + let outputs = outputs_of + .get(&query) + .into_iter() + .flatten() .map(|answering| { answering .iter() @@ -174,18 +181,20 @@ pub(crate) fn copies_in_equivalent_contexts( .collect::>>() }) .collect::>>()?; - Some(( - outputs, - inputs + Some(QuerySignature { + answering_executions_per_output: outputs, + exact_only_groups: inputs .physical .logical_candidates .exact_only_groups_of_query .get(query.0), - inputs.entries[query.0].range_span_ms(), - target_of_query[query.0], - )) + range_span_ms: inputs.entries[query.0].range_span_ms(), + accuracy_target_index: target_of_query[query.0], + }) }; - let key_of = |context: ExecutionContextId| -> Option> { + // `context`'s key, and its queries in the order of their signatures in + // it. + let key_of = |context: ExecutionContextId| -> Option<(ContextKey<'_>, Vec)> { let executions = executions_of[&context] .iter() .map(|&index| { @@ -218,61 +227,77 @@ pub(crate) fn copies_in_equivalent_contexts( .get(&context) .into_iter() .flatten() - .map(|&query| signature(context, query)) + .map(|&query| Some((signature(context, query)?, query))) .collect::>>()?; queries.sort(); - Some(ContextKey { - executions, - queries, - }) + let (queries, in_signature_order) = queries.into_iter().unzip(); + Some(( + ContextKey { + executions, + queries, + }, + in_signature_order, + )) }; - // Whether every cost of `copy` is the same multiple of `original`'s, - // their executions and edges paired in order (equal keys). + // Whether every cost of `copy` is the same multiple α of `original`'s, + // their executions and edges paired in order (equal keys), to + // `COST_RELATIVE_TOLERANCE` of the copy's largest cost, α taken from + // that pair. An edge cost is the consumer's cost through one producer + // less its own, a small difference of large costs that keeps their + // rounding: 2.8110024e-9 − 2.8110047e-9 in a 60 s context and the same + // over 5 in a 300 s one are −2.3556e-15 and −4.7112e-16, a ratio of + // 0.200000000026, off by far more than 1e-12 of themselves but not of + // the context's costs. let proportional_costs = |original: ExecutionContextId, copy: ExecutionContextId| { - let mut alpha: Option = None; - let mut proportional = |a: f64, b: f64| match (a == 0.0, b == 0.0) { - (true, true) => true, - (false, false) => { - let ratio = *alpha.get_or_insert(b / a); - (b - ratio * a).abs() <= COST_RELATIVE_TOLERANCE * b.abs() + let mut pairs: Vec<(f64, f64)> = Vec::new(); + for (&a, &b) in executions_of[&original].iter().zip(&executions_of[©]) { + if let (Ok(cost_a), Ok(cost_b)) = (&costs.per_execution[a], &costs.per_execution[b]) { + pairs.push((*cost_a, *cost_b)); } - _ => false, + for input in 0..selectable.input_count(a) { + for (&edge_a, &edge_b) in selectable + .edges_into(a, input) + .iter() + .zip(selectable.edges_into(b, input)) + { + if let (Some(cost_a), Some(cost_b)) = + (costs.per_edge[edge_a], costs.per_edge[edge_b]) + { + pairs.push((cost_a, cost_b)); + } + } + } + } + let Some(&(largest_a, largest_b)) = + pairs.iter().max_by(|x, y| x.1.abs().total_cmp(&y.1.abs())) + else { + return true; }; - executions_of[&original] + if largest_a == 0.0 || largest_b == 0.0 { + return pairs.iter().all(|&(a, b)| a == 0.0 && b == 0.0); + } + let alpha = largest_b / largest_a; + let tolerance = COST_RELATIVE_TOLERANCE * largest_b.abs(); + pairs .iter() - .zip(&executions_of[©]) - .all(|(&a, &b)| { - let own = match (&costs.per_execution[a], &costs.per_execution[b]) { - (Ok(cost_a), Ok(cost_b)) => proportional(*cost_a, *cost_b), - _ => true, - }; - own && (0..selectable.input_count(a)).all(|input| { - selectable - .edges_into(a, input) - .iter() - .zip(selectable.edges_into(b, input)) - .all(|(&edge_a, &edge_b)| { - match (costs.per_edge[edge_a], costs.per_edge[edge_b]) { - (Some(cost_a), Some(cost_b)) => proportional(cost_a, cost_b), - _ => true, - } - }) - }) - }) + .all(|&(a, b)| (b - alpha * a).abs() <= tolerance) }; - let mut originals_of_key: HashMap, Vec> = HashMap::new(); + // Per key, the originals, each with its queries in signature order. + let mut originals_of_key: HashMap, Vec<(ExecutionContextId, Vec)>> = + HashMap::new(); for © in executions_of.keys() { - let Some(key) = key_of(copy) else { + let Some((key, copy_queries)) = key_of(copy) else { continue; }; let originals = originals_of_key.entry(key).or_default(); - let Some(&original) = originals + let Some((original, original_queries)) = originals .iter() - .find(|&&original| proportional_costs(original, copy)) + .find(|(original, _)| proportional_costs(*original, copy)) else { - originals.push(copy); + originals.push((copy, copy_queries)); continue; }; + let original = *original; for (&a, &b) in executions_of[&original].iter().zip(&executions_of[©]) { shared.original_of_execution[b] = a; for input in 0..selectable.input_count(b) { @@ -293,17 +318,7 @@ pub(crate) fn copies_in_equivalent_contexts( ); } // Equal keys: the queries pair up in signature order. - let by_signature = |context: ExecutionContextId| { - let mut queries: Vec<(QuerySignature<'_>, QueryId)> = queries_of - .get(&context) - .into_iter() - .flatten() - .map(|&query| (signature(context, query).expect("keyed"), query)) - .collect(); - queries.sort(); - queries - }; - for ((_, a), (_, b)) in by_signature(original).into_iter().zip(by_signature(copy)) { + for (&a, b) in original_queries.iter().zip(copy_queries) { shared.original_of_query[b.0] = a; } } diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index d84939e18..3d3566dc1 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -196,13 +196,7 @@ fn selection_inputs( raw_data_source: HashMap::new(), unbounded_raw_data_on_disk_cost_per_second: HashMap::new(), }; - costs.least_need_ms_per_execution = least_need_ms_per_execution( - &selectable, - &costs.per_edge, - &costs.retention_need_ms_per_edge, - &costs.newest_output_retention_ms, - entries, - ); + costs.least_need_ms_per_execution = least_need_ms_per_execution(&selectable, &costs, entries); (selectable, costs) } @@ -475,13 +469,8 @@ fn a_stored_result_read_at_its_newest_output_retains_one_output() { costs .newest_output_retention_ms .insert(PhysicalCandidateId(1), newest_output_ms); - costs.least_need_ms_per_execution = least_need_ms_per_execution( - &selectable, - &costs.per_edge, - &costs.retention_need_ms_per_edge, - &costs.newest_output_retention_ms, - &entries, - ); + costs.least_need_ms_per_execution = + least_need_ms_per_execution(&selectable, &costs, &entries); } solve_selection(&SelectionInputs { physical: &physical, @@ -1084,3 +1073,65 @@ fn a_context_equivalent_to_another_shares_its_variables() { .enumerate() .all(|(index, &original)| original == index)); } + +/// The same query `m[5m]` every 60 s and every 300 s, each cost in the +/// 300 s context a fifth of the 60 s one's, except the scan-to-range +/// edge: a difference of the range's cost through the scan and its own, +/// as RollupHeavy-8users' flows component computes it, +/// 2.811002367077395e-9 − 2.8110047226529567e-9 (−2.3556e-15) in the 60 s +/// context and 5.622004734154791e-10 − 5.622009445305915e-10 (−4.7112e-16) +/// in the 300 s one. The two differences are in the ratio 0.200000000026, +/// off by more than 1e-12 of themselves but not of the contexts' costs, +/// so the contexts are copies. +#[test] +fn contexts_whose_small_edge_costs_differ_by_rounding_are_copies() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + ], + ); + let entries = [every(60_000), every(300_000)]; + let (selectable, mut costs) = selection_inputs(&physical, &entries, &[1.0, 1.5, 1.0], &[], &[]); + let executions = selectable.executions(); + let every_300s = |index: usize| { + matches!( + selectable.context(executions[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 300_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_300s(index) { + *cost = Ok(cost.clone().unwrap() / 5.0); + } + } + let scan_to_range = edge(&physical, 0, 2); + for (position, selectable_edge) in selectable.edges().iter().enumerate() { + if selectable_edge.edge == scan_to_range { + costs.per_edge[position] = Some(match every_300s(selectable_edge.consumer) { + true => 5.622004734154791e-10 - 5.622009445305915e-10, + false => 2.811002367077395e-9 - 2.8110047226529567e-9, + }); + } + } + let shared = copies_in_equivalent_contexts( + &SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }, + &vec![true; executions.len()], + ); + assert_eq!(shared.original_of_query, vec![QueryId(0), QueryId(0)]); + for (index, &original) in shared.original_of_execution.iter().enumerate() { + assert_eq!(original != index, every_300s(index)); + } +} diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs index 367126268..e3e9fed42 100644 --- a/crates/stage3-plan-selection/src/milp/retention.rs +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -96,9 +96,9 @@ pub(crate) fn retention_constraints( /// The least $h_v$ (R) allows for a selection, in ms, of each candidate /// with a retention cost that `edges` (selected, by position in /// `SelectableExecutions::edges`) or `outputs` (the query outputs and the -/// executions answering them, by position) read, at least its newest -/// output (`ExecutionCosts::newest_output_retention_ms`): what the MILP's optimum -/// retains when $\pi_v > 0$, and what the plan retains. +/// executions answering them, by position) read, each read's need at +/// least its newest output (`ExecutionCosts::retention_ms_for_need`): what +/// the MILP's optimum retains when $\pi_v > 0$, and what the plan retains. pub(crate) fn least_retention_ms( inputs: &SelectionInputs<'_>, edges: impl IntoIterator, @@ -119,15 +119,10 @@ pub(crate) fn least_retention_ms( }; let v = executions[index].candidate; if let (true, Some(need_ms), true) = (selected, need_ms, retaining.contains_key(&v)) { + let need_ms = inputs.costs.retention_ms_for_need(v, need_ms); let most = retention_ms.entry(v).or_default(); *most = (*most).max(need_ms); } } - // At least the newest output, which a store always keeps. - for (v, most) in &mut retention_ms { - if let Some(&newest_ms) = inputs.costs.newest_output_retention_ms.get(v) { - *most = (*most).max(newest_ms); - } - } retention_ms } diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index a36f90842..271395327 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -15,6 +15,7 @@ use std::collections::{BTreeMap, HashMap}; use super::model::{has_one_reader, selection_cost_per_second, SelectionInputs}; +use super::COST_RELATIVE_TOLERANCE; use crate::selectable::QueryId; /// How many times the greedy plan is rebuilt without an execution that @@ -52,6 +53,11 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec 0.0, } }; + // Looser than `COST_RELATIVE_TOLERANCE` on purpose: the bound and each + // execution's least cost are sums over many executions, and leaving + // out an execution the optimum needs cannot be undone, so an execution + // is left out only when it is clearly dearer (by 1e-9 of the bound; + // 1e-21/s when the bound is 0). let tolerance = 1e-9 * bound.abs().max(1e-12); (0..count) .map(|index| { @@ -180,7 +186,9 @@ fn cheapest_trees(inputs: &SelectionInputs<'_>, excluded: &[bool]) -> (Vec, } } } - if cost < tree_cost[v] - 1e-12 * cost.abs().max(1e-300) { + // Cheaper by more than rounding (the smallest positive + // magnitude when the cost is 0). + if cost < tree_cost[v] - COST_RELATIVE_TOLERANCE * cost.abs().max(1e-300) { tree_cost[v] = cost; chosen[v] = reads; changed = true; diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 253181b94..7d1311fe6 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -8,8 +8,6 @@ //! (`docs/design_docs/concepts/memo-groups-properties-requirements.md`, //! section 5, "Cost and retention"). -use std::collections::HashMap; - use asap_types::ir::Operator; use asap_types::plan::time_window::{InputTimeAccess, TimeIndexedPlan}; use asap_types::stages::{ @@ -17,6 +15,7 @@ use asap_types::stages::{ }; use asap_types::workload::QueryWorkloadEntry; +use crate::cost::ExecutionCosts; use crate::selectable::{Reader, SelectableExecutions}; /// $\operatorname{need}_{\kappa,u,c,i}$: how far back from the evaluation @@ -55,40 +54,34 @@ pub(crate) fn edge_scan_read_ms( read_back(physical, edge, |_| 0) } -/// The least need, in ms, over each execution's costed reads -/// (`SelectableExecutions::reads_back`: an edge out of it `per_edge` costs, or a query output -/// it may answer), by position in [`SelectableExecutions::executions`]; -/// `None` without a bounded one; at least its candidate's -/// `newest_output_retention_ms` (Δ, the newest output a store always -/// keeps). (F) gives a selected execution a use and (R) then retains at -/// least that: the greedy bound charges it, and the MILP has $h_v \ge$ it -/// $\cdot x$. +/// The least retention, in ms, over each execution's costed reads +/// (`SelectableExecutions::reads_back`: an edge out of it `costs.per_edge` +/// costs, or a query output it may answer), each read's need at least its +/// candidate's newest output (`ExecutionCosts::retention_ms_for_need`), by +/// position in [`SelectableExecutions::executions`]; `None` without a +/// bounded one. (F) gives a selected execution a use and (R) then retains +/// at least that: the greedy bound charges it, and the MILP has +/// $h_v \ge$ it $\cdot x$. pub(crate) fn least_need_ms_per_execution( selectable: &SelectableExecutions, - per_edge: &[Option], - need_ms_per_edge: &[Option], - newest_output_retention_ms: &HashMap, + costs: &ExecutionCosts, entries: &[QueryWorkloadEntry], ) -> Vec> { let mut least_need_ms = vec![None; selectable.executions().len()]; - for (index, reader, need_ms) in selectable.reads_back(need_ms_per_edge, entries) { + for (index, reader, need_ms) in + selectable.reads_back(&costs.retention_need_ms_per_edge, entries) + { let costed = match reader { - Reader::Edge(position) => per_edge[position].is_some(), + Reader::Edge(position) => costs.per_edge[position].is_some(), Reader::Output(_) => true, }; if let (true, Some(need_ms)) = (costed, need_ms) { + let need_ms = + costs.retention_ms_for_need(selectable.executions()[index].candidate, need_ms); let least: &mut Option = &mut least_need_ms[index]; *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); } } - for (least, execution) in least_need_ms.iter_mut().zip(selectable.executions()) { - if let (Some(least), Some(&newest_ms)) = ( - least.as_mut(), - newest_output_retention_ms.get(&execution.candidate), - ) { - *least = (*least).max(newest_ms); - } - } least_need_ms } diff --git a/crates/stage3-plan-selection/src/selectable.rs b/crates/stage3-plan-selection/src/selectable.rs index ea635ab99..06d399316 100644 --- a/crates/stage3-plan-selection/src/selectable.rs +++ b/crates/stage3-plan-selection/src/selectable.rs @@ -6,7 +6,7 @@ //! output the executions that may answer it ([`SelectableExecutions::new`]). //! //! - A query is evaluated in the evaluation context of its grid -//! ([`evaluation_grid`]); queries on one grid share it, a range query's +//! (`QueryWorkloadEntry::evaluation_grid`); queries on one grid share it, a range query's //! steps included. //! - A candidate with $\ell$ = None or `OneEvaluation` runs once per context //! that selects it ($x_{\kappa,v}$), the context of its consumer; at @@ -114,7 +114,7 @@ pub(crate) struct SelectableExecutions { /// answer it: $z_{q,v}$ exists only for them. answering: Vec<(QueryId, Vec)>, /// Per query, by [`QueryId`], the evaluation context it is evaluated - /// in: `Evaluation { grid: evaluation_grid(entry) }`, no request + /// in: `Evaluation { grid: entry.evaluation_grid() }`, no request /// anchor; `None` for a query with no output reference. Its answering /// executions may run elsewhere (an `AcrossEvaluations` one in its own /// `Materialized(v)`), so this is not read off them. @@ -155,7 +155,7 @@ impl SelectableExecutions { for &(query, group) in outputs { let context = selectable.context_id( ExecutionContext::Evaluation { - grid: evaluation_grid(&entries[query.0]), + grid: entries[query.0].evaluation_grid(), }, None, ); @@ -494,12 +494,3 @@ fn reads_of_groups( entries, ) } - -/// The evaluation grid of `entry`'s query, the grid of its context -/// (`QueryWorkloadEntry::evaluation_grid_ms`), anchored at the epoch. -pub(crate) fn evaluation_grid(entry: &QueryWorkloadEntry) -> EvaluationGrid { - EvaluationGrid { - period_ms: entry.evaluation_grid_ms(), - phase_ms: 0, - } -} diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 40945fe0d..7fa100936 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -45,6 +45,7 @@ impl Operator { } } + /// `self` with `f` applied to each child, in [`Self::children`] order. /// `f` may change the reference type, e.g. from `Rc` to a /// node id. pub fn map_children(&self, f: impl FnMut(&C) -> D) -> Operator { diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 2318fdd8c..66f29d0d0 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -251,9 +251,13 @@ impl NonASAPOp { } /// Rebuild this operator with `f` applied to every child, including the - /// operator nodes referenced from scalar expressions. Every other field - /// is cloned. `f` may change the reference type, e.g. from - /// `Rc` to a node id. + /// operator nodes referenced from scalar expressions, in + /// [`Self::children`] order (direct inputs first), so a caller may hand + /// out replacements by position. Every other field is cloned. `f` may + /// change the reference type, e.g. from `Rc` to a node id. + /// + /// Example: `Project { child: q, cols: [q.value + scalar(c)] }` calls `f` + /// on `q`, then on `c`. pub(crate) fn map_children(&self, mut f: impl FnMut(&C) -> D) -> NonASAPOp { use NonASAPOp::*; fn map_pred(p: &Predicate, f: &mut impl FnMut(&C) -> D) -> Predicate { @@ -287,13 +291,14 @@ impl NonASAPOp { }, PromqlVectorFromScalar(e) => PromqlVectorFromScalar(e.map_operator_refs(&mut f)), Filter { pred, child } => { - let pred = map_pred(pred, &mut f); + let child = f(child); Filter { - pred, - child: f(child), + pred: map_pred(pred, &mut f), + child, } } Project { cols, child } => { + let child = f(child); let cols = cols .iter() .map(|c| ProjectItem { @@ -301,10 +306,7 @@ impl NonASAPOp { expr: c.expr.map_operator_refs(&mut f), }) .collect(); - Project { - cols, - child: f(child), - } + Project { cols, child } } Aggregate { reduction, @@ -314,6 +316,7 @@ impl NonASAPOp { having, child, } => { + let child = f(child); let filters = filters .iter() .map(|p| p.as_ref().map(|p| map_pred(p, &mut f))) @@ -325,7 +328,7 @@ impl NonASAPOp { output_names: output_names.clone(), filters, having, - child: f(child), + child, } } Join { @@ -334,12 +337,12 @@ impl NonASAPOp { left, right, } => { - let pred = map_pred(pred, &mut f); + let (left, right) = (f(left), f(right)); Join { kind: kind.clone(), - pred, - left: f(left), - right: f(right), + pred: map_pred(pred, &mut f), + left, + right, } } SetOp { @@ -365,11 +368,11 @@ impl NonASAPOp { partition_by, child, } => { - let keys = map_keys(keys, &mut f); + let child = f(child); Sort { - keys, + keys: map_keys(keys, &mut f), partition_by: partition_by.clone(), - child: f(child), + child, } } Limit { @@ -403,6 +406,7 @@ impl NonASAPOp { output_name, child, } => { + let child = f(child); let args = args.iter().map(|e| e.map_operator_refs(&mut f)).collect(); let order_by = map_keys(order_by, &mut f); SQLWindowFunc { @@ -412,7 +416,7 @@ impl NonASAPOp { order_by, frame: frame.clone(), output_name: output_name.clone(), - child: f(child), + child, } } TimeRange { range, kind, child } => TimeRange { @@ -425,11 +429,11 @@ impl NonASAPOp { child: f(child), }, PromqlRelabel { dst, value, child } => { - let value = value.map_operator_refs(&mut f); + let child = f(child); PromqlRelabel { dst: dst.clone(), - value, - child: f(child), + value: value.map_operator_refs(&mut f), + child, } } PromqlInfoEnrich { selector, child } => PromqlInfoEnrich { diff --git a/crates/types/src/plan/materialized_view/mod.rs b/crates/types/src/plan/materialized_view/mod.rs index 3d9f8d7f1..e3a8182ae 100644 --- a/crates/types/src/plan/materialized_view/mod.rs +++ b/crates/types/src/plan/materialized_view/mod.rs @@ -77,6 +77,7 @@ use thiserror::Error; use super::physical_asap_dag::{PhysicalASAPDAGValidationError, PhysicalASAPNodeId, ViewCoverage}; use super::time_window::{self, TimeIndexedPlan, TimeWindow}; use crate::ir::operator::arguments::Source; +use crate::stages::EvaluationGrid; /// One materialized view of a DAG: what its stored state is, what it is /// refreshed from, when, and for how long it is retained. @@ -195,6 +196,19 @@ impl RefreshWindow { } } +/// The times a refresh window ends at: every `width_ms`, at `phase_ms`. +/// +/// Example: `RefreshWindow { width_ms: 300_000, phase_ms: 0 }`, the +/// refresh of `sum(TimeRange(5m))`, ends every 5 min at phase 0. +impl From for EvaluationGrid { + fn from(window: RefreshWindow) -> Self { + EvaluationGrid { + period_ms: window.width_ms, + phase_ms: window.phase_ms, + } + } +} + /// The views refreshed at ingestion that share one refresh window /// ([`PhysicalASAPDAG::ingestion_refresh_groups`]). A runtime refreshes /// them with one ingestion run per window `(e - W, e]`, whose roots are its diff --git a/crates/types/src/stages/logical/memo_candidates.rs b/crates/types/src/stages/logical/memo_candidates.rs index 537fbf4d6..bdb3b9090 100644 --- a/crates/types/src/stages/logical/memo_candidates.rs +++ b/crates/types/src/stages/logical/memo_candidates.rs @@ -4,8 +4,6 @@ use std::collections::BTreeSet; -use crate::plan::materialized_view::RefreshWindow; - use crate::stages::{ CandidateForMemoGroupId, LogicalEdge, Memo, MemoGroupId, PrunedLogicalCandidate, }; @@ -23,19 +21,6 @@ pub struct EvaluationGrid { pub phase_ms: u64, } -/// The times a refresh window ends at: every `width_ms`, at `phase_ms`. -/// -/// Example: `RefreshWindow { width_ms: 300_000, phase_ms: 0 }`, the -/// refresh of `sum(TimeRange(5m))`, ends every 5 min at phase 0. -impl From for EvaluationGrid { - fn from(window: RefreshWindow) -> Self { - EvaluationGrid { - period_ms: window.width_ms, - phase_ms: window.phase_ms, - } - } -} - /// Stage 1, Pass 2's output. #[derive(Debug, Clone)] pub struct LogicalASAPMemoCandidates { @@ -60,7 +45,9 @@ pub struct LogicalASAPMemoCandidates { /// Q0 = `quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, /// m[5m])` and Q1 = the same ratio `> 1`, both relative 0.05: Q0 may /// read the DDSketch readouts, while Q1's comparison has no contract, - /// so its set holds the ratio group and every group below it. (Both + /// so its set holds the ratio group and every group below it, except + /// below an input whose error does not reach its consumer's output (a + /// `count`'s input: an exact `count(q)` lists the count, not `q`). (Both /// list the rows below a sketch build, which has no contract.) A query /// whose whole-result failure probability has no per-value one ($n_q$ /// unknown) also lists each readout group another query gave a level, @@ -77,7 +64,9 @@ impl LogicalASAPMemoCandidates { /// (it has sketch parameters θ) and its group is one the query may read /// only exactly ([`Self::exact_only_groups_of_query`]). A readout's own /// candidate has no θ, but the sketch states it reads are in the same - /// list, as every group below a listed one is. + /// list, as every group below a listed one is, except below an input + /// whose error does not reach its consumer's output (a `count`'s; a + /// readout's state input is not one). /// /// Example: `quantile_over_time(0.9, m[5m])` at a relative target lists /// the quantile and every group below it; its KLL state builds (θ) are diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs index 863fff596..e3b238036 100644 --- a/crates/types/src/stages/physical/execution_context.rs +++ b/crates/types/src/stages/physical/execution_context.rs @@ -38,6 +38,20 @@ impl ExecutionContext { width_ms: ONE_SECOND_MS, phase_ms: 0, }); + + /// Whether this context runs outside every evaluation context, so + /// every evaluation context reading its executions shares them: the + /// ingestion pipeline and a candidate's own materialized computation. + /// + /// Example: a chain materialized at ingestion time, read by queries + /// every 60 s and every 300 s: its `IngestionPipeline` execution is one + /// for both, true; an `Evaluation` context: false. + pub fn is_outside_evaluations(&self) -> bool { + match self { + Self::Evaluation { .. } => false, + Self::IngestionPipeline(_) | Self::Materialized(_) => true, + } + } } /// Whether a `TimeShift` anchored with `at`, read by `queries` (those on diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index c1aeef8be..2e5c64d2f 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -2,6 +2,7 @@ use crate::accuracy::AccuracyTarget; use crate::ir::operator::AtModifier; +use crate::stages::EvaluationGrid; use serde::{Deserialize, Serialize}; use super::data::validate_rate; @@ -427,6 +428,17 @@ impl QueryWorkloadEntry { .unwrap_or(1) } + /// The evaluation grid of the query's evaluation context: every + /// [`Self::evaluation_grid_ms`], anchored at the epoch (phase 0). + /// + /// Example: every 20 s at phase 5 s: `{ period_ms: 5_000, phase_ms: 0 }`. + pub fn evaluation_grid(&self) -> EvaluationGrid { + EvaluationGrid { + period_ms: self.evaluation_grid_ms(), + phase_ms: 0, + } + } + /// How far back from a request's end its evaluations reach, in ms: its /// range-evaluation span, 0 for one evaluation per run. The output it /// reads directly is read back that far. From 0b5ec46538495da6ac48fc6d667715968a83b16f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:19:22 +0000 Subject: [PATCH 23/23] fix(stage3,types,cost,accuracy): round-6 review fixes: context copies compare kept costs only, IN subquery order, producers from the payload (#1207 PR 7) - E1-b: proportional_costs pairs only kept executions and edges between kept ones, so a left-out execution no longer sets the copy tolerance; alpha must be positive. Test: a left-out range costing 1e12 merged two contexts (optimum 2.5 instead of 2.421). - N1 gap: ScalarExpr::map_parts maps an IN's subquery before its left side, as operator_refs lists them; a test checks map_children follows children() for every operator with scalar references. - L-1: least_need_ms_per_execution -> least_retention_ms_per_execution. - A2: plan_cost reads producers from PhysicalASAPDAGNode::producers instead of sorting edges. A3: unretained nodes built once. A4: outputs_of[&query]. L-2: input_error_reaches_output is pub(crate) and indexes directly. --- crates/accuracy/src/error_contract/mod.rs | 6 +- crates/cost/src/plan/cost.rs | 27 ++--- crates/stage3-plan-selection/src/cost.rs | 31 +++-- .../src/milp/equivalent_contexts.rs | 60 ++++++---- .../src/milp/model/tests.rs | 73 +++++++++++- .../src/milp/retention.rs | 9 +- .../src/milp/upper_bound.rs | 2 +- crates/stage3-plan-selection/src/retention.rs | 12 +- crates/types/src/ir/operator/non_asap.rs | 107 ++++++++++++++++++ crates/types/src/ir/scalar/mod.rs | 14 ++- crates/types/src/plan/physical_asap_dag.rs | 14 +++ 11 files changed, 281 insertions(+), 74 deletions(-) diff --git a/crates/accuracy/src/error_contract/mod.rs b/crates/accuracy/src/error_contract/mod.rs index 2ad2a9a19..304dd7814 100644 --- a/crates/accuracy/src/error_contract/mod.rs +++ b/crates/accuracy/src/error_contract/mod.rs @@ -344,10 +344,8 @@ impl ErrorContract { /// Example: `count` outputs how many values it reads, whatever their /// error ($\beta = 0$): false; `sum` passes its input's relative error /// on ($\beta = 1$): true. - pub fn input_error_reaches_output(&self, input: usize) -> bool { - self.error_factor_per_input - .get(input) - .is_none_or(|&factor| factor > 0.0) + pub(crate) fn input_error_reaches_output(&self, input: usize) -> bool { + self.error_factor_per_input[input] > 0.0 } } diff --git a/crates/cost/src/plan/cost.rs b/crates/cost/src/plan/cost.rs index 9d9e5d1ec..47922c71c 100644 --- a/crates/cost/src/plan/cost.rs +++ b/crates/cost/src/plan/cost.rs @@ -105,21 +105,16 @@ pub fn plan_cost( // usage and cost ([`node_resource_usage_and_cost`]), children first. let multipliers = computations(dag, entries, settings.non_repeating_query_horizon_seconds) .map_err(|error| (first, error))?; - // Each node's producers in its payload's input order (its children, - // then the nodes its scalar expressions refer to), whatever order the - // DAG lists its edges in: its first producer is its child. Example: - // `q + scalar(count(q))` as a `Project` over `q` reads `q` first, even - // when the edge from the count is listed first. - let mut producers_of: HashMap> = HashMap::new(); - for e in &dag.edges { - producers_of.entry(e.consumer).or_default().push(e.producer); - } - for (id, producers) in &mut producers_of { - let inputs = nodes[id].payload.children(); - producers.sort_by_key(|producer| inputs.iter().position(|&input| input == producer)); - } + // Each node's producers in its payload's input order, whatever order + // the DAG lists its edges in (`PhysicalASAPDAGNode::producers`): its + // first producer is its child. + let producers_of: HashMap> = dag + .nodes + .iter() + .map(|node| (node.id, node.producers())) + .collect(); let first_input_of = |id: PhysicalASAPNodeId| { - let input = producers_of.get(&id)?.first()?; + let input = producers_of[&id].first()?; Some(&nodes[input].payload) }; // Each node keeping raw rows at ingestion, not stored: what an @@ -139,9 +134,7 @@ pub fn plan_cost( let mut costed: HashMap> = HashMap::new(); let mut one_batch: HashMap> = HashMap::new(); for node in &dag.nodes { - let views: Vec> = producers_of - .get(&node.id) - .map_or(&[][..], Vec::as_slice) + let views: Vec> = producers_of[&node.id] .iter() .map(|p| { let read = match reads_one_batch_of(node, nodes[p], first_input_of(*p)) { diff --git a/crates/stage3-plan-selection/src/cost.rs b/crates/stage3-plan-selection/src/cost.rs index a2fa2ebd0..2af18c0d0 100644 --- a/crates/stage3-plan-selection/src/cost.rs +++ b/crates/stage3-plan-selection/src/cost.rs @@ -72,7 +72,7 @@ use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::plan::candidate_node; use crate::retention::{ - edge_retention_need_ms, edge_scan_read_ms, least_need_ms_per_execution, ms_to_seconds, + edge_retention_need_ms, edge_scan_read_ms, least_retention_ms_per_execution, ms_to_seconds, }; use crate::selectable::{Execution, SelectableExecutions}; @@ -110,8 +110,8 @@ pub(crate) struct ExecutionCosts { pub(crate) newest_output_retention_ms: HashMap, /// The least retention, in ms, over each execution's costed reads /// ([`Self::retention_ms_for_need`] of each), by position in - /// `SelectableExecutions::executions` ([`least_need_ms_per_execution`]). - pub(crate) least_need_ms_per_execution: Vec>, + /// `SelectableExecutions::executions` ([`least_retention_ms_per_execution`]). + pub(crate) least_retention_ms_per_execution: Vec>, /// The source of each query-time `Scan`: its retention is raw data on /// disk, which a plan keeps once per source for the selected scan that /// keeps the most, as `plan_cost` charges it ($r_s$). @@ -171,8 +171,12 @@ pub(crate) fn execution_costs( node.retention_ms = retention_ms; node }; - let retains_rows: Vec = (0..count) - .map(|v| node_retaining(PhysicalCandidateId(v), None).retains_rows()) + let unretained_nodes: Vec = (0..count) + .map(|v| node_retaining(PhysicalCandidateId(v), None)) + .collect(); + let retains_rows: Vec = unretained_nodes + .iter() + .map(PhysicalASAPDAGNode::retains_rows) .collect(); let query_time_scan: Vec = (0..count) .map(|v| physical.is_query_time_scan(PhysicalCandidateId(v))) @@ -201,7 +205,7 @@ pub(crate) fn execution_costs( .filter(|&v| retains_rows[v]) .map(PhysicalCandidateId) .map(|v| { - let node = node_retaining(v, None); + let node = &unretained_nodes[v.0]; let window_ms = match node.output_state.timing { ExecutionTiming::IngestionTime => ingestion_window_ms(physical, v), ExecutionTiming::QueryTime => None, @@ -268,9 +272,13 @@ pub(crate) fn execution_costs( } let baseline_scan_read_of = |v: PhysicalCandidateId| baseline_scan_read_ms.get(&v).copied().flatten(); - let nodes: Vec = (0..count) - .map(PhysicalCandidateId) - .map(|v| node_retaining(v, most_read_ms.get(&v).copied().flatten())) + let nodes: Vec = unretained_nodes + .into_iter() + .enumerate() + .map(|(v, mut node)| { + node.retention_ms = most_read_ms.get(&PhysicalCandidateId(v)).copied().flatten(); + node + }) .collect(); // Each group's expression as a payload, for its logical output. let group_payloads: Vec = memo @@ -500,11 +508,12 @@ pub(crate) fn execution_costs( retention_cost_per_retained_second, retention_need_ms_per_edge, newest_output_retention_ms, - least_need_ms_per_execution: Vec::new(), + least_retention_ms_per_execution: Vec::new(), raw_data_source, unbounded_raw_data_on_disk_cost_per_second, }; - costs.least_need_ms_per_execution = least_need_ms_per_execution(selectable, &costs, entries); + costs.least_retention_ms_per_execution = + least_retention_ms_per_execution(selectable, &costs, entries); costs } diff --git a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs index 2dd4dcc90..f48352bb0 100644 --- a/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs +++ b/crates/stage3-plan-selection/src/milp/equivalent_contexts.rs @@ -41,7 +41,7 @@ enum ExecutionInContext { /// What two equivalent contexts have alike, up to their costs' common /// multiple $\alpha$: per execution (ascending, so by candidate) its -/// candidate, whether it is kept and costed, its least retention need, and +/// candidate, whether it is kept and costed, its least retention, and /// per input the memo edges into it from each producer, costed or not; and /// the sorted signatures of its queries ([`QuerySignature`]). #[derive(Debug, PartialEq, Eq, Hash)] @@ -56,7 +56,7 @@ struct ExecutionKey { candidate: PhysicalCandidateId, kept: bool, costed: bool, - least_need_ms: Option, + least_retention_ms: Option, /// Per input, each edge into it: (memo edge, producer, costed). inputs: Vec>, } @@ -84,22 +84,25 @@ struct QuerySignature<'a> { /// same execution outside every evaluation context, and queries that pair /// up with the same output executions, exact-only groups, span and /// accuracy target, so the accuracy constraint holds alike in both) and -/// every cost in one is the same multiple $\alpha$ of the other's, to -/// [`COST_RELATIVE_TOLERANCE`] of the copy's largest cost. Each query -/// belongs to its own evaluation context +/// every cost the MILP reads in one (of a kept execution, or of an edge +/// between two kept ones) is the same multiple $\alpha > 0$ of the +/// other's, to [`COST_RELATIVE_TOLERANCE`] of the copy's largest such +/// cost. Each query belongs to its own evaluation context /// (`SelectableExecutions::evaluation_context`), wherever its answering /// executions run. Contexts are bucketed by key, so only the cost check /// compares two contexts. /// -/// Exact: from an optimal plan, give both contexts the selection of the -/// one whose own cost $A$ or $B$ is lower, say $A$. The copied context -/// then costs $\alpha A \le \alpha B$, the retention $h_v$ and raw data -/// $r_s$ (each the most of their readers' needs, equal in both copies) do -/// not grow, and an execution outside them that only the other selection -/// read is left out at no cost; with $B \le A$ symmetrically. So some -/// optimum selects alike in both, and sharing their variables keeps it, -/// to the tolerance the costs are compared to (each copied cost within -/// 1e-12 of the context's largest cost of $\alpha$ times its original). +/// Why some optimum survives: from an optimal plan, give both contexts the +/// selection of the one whose own cost $A$ or $B$ is lower, say $A$. The +/// copied context then costs $\alpha A \le \alpha B$, the retention $h_v$ +/// and raw data $r_s$ (each the most of their readers' needs, equal in both +/// copies) do not grow, and an execution outside them that only the other +/// selection read is left out at no cost; with $B \le A$ symmetrically. So +/// some optimum selects alike in both, and sharing their variables keeps +/// it, to the tolerance the costs are compared to: each copied cost the +/// MILP reads (of a kept execution, or of an edge between two kept ones) +/// within 1e-12 of the largest of them of $\alpha$ times its original. The +/// exchange needs $\alpha > 0$. /// /// Example: the same queries every 60 s and every 300 s (one user each): /// the 300 s context's executions cost 1/5 of the 60 s ones' and share @@ -170,10 +173,9 @@ pub(crate) fn copies_in_equivalent_contexts( }) .collect(); let signature = |context: ExecutionContextId, query: QueryId| -> Option> { - let outputs = outputs_of - .get(&query) - .into_iter() - .flatten() + // A query has an evaluation context exactly when it has an output reference. + let outputs = outputs_of[&query] + .iter() .map(|answering| { answering .iter() @@ -218,7 +220,7 @@ pub(crate) fn copies_in_equivalent_contexts( candidate: executions[index].candidate, kept: kept[index], costed: costs.per_execution[index].is_ok(), - least_need_ms: costs.least_need_ms_per_execution[index], + least_retention_ms: costs.least_retention_ms_per_execution[index], inputs, }) }) @@ -239,10 +241,13 @@ pub(crate) fn copies_in_equivalent_contexts( in_signature_order, )) }; - // Whether every cost of `copy` is the same multiple α of `original`'s, - // their executions and edges paired in order (equal keys), to - // `COST_RELATIVE_TOLERANCE` of the copy's largest cost, α taken from - // that pair. An edge cost is the consumer's cost through one producer + // Whether every cost of `copy` the MILP reads is the same multiple + // α > 0 of `original`'s, their executions and edges paired in order + // (equal keys), to `COST_RELATIVE_TOLERANCE` of the copy's largest such + // cost, α taken from that pair. Only kept executions and edges between + // two kept ones count (equal keys keep them alike in both): a left-out + // execution has no cost in the MILP, and one costing 1e12 would + // otherwise allow each kept cost an error of 1. An edge cost is the consumer's cost through one producer // less its own, a small difference of large costs that keeps their // rounding: 2.8110024e-9 − 2.8110047e-9 in a 60 s context and the same // over 5 in a 300 s one are −2.3556e-15 and −4.7112e-16, a ratio of @@ -251,6 +256,9 @@ pub(crate) fn copies_in_equivalent_contexts( let proportional_costs = |original: ExecutionContextId, copy: ExecutionContextId| { let mut pairs: Vec<(f64, f64)> = Vec::new(); for (&a, &b) in executions_of[&original].iter().zip(&executions_of[©]) { + if !kept[a] { + continue; + } if let (Ok(cost_a), Ok(cost_b)) = (&costs.per_execution[a], &costs.per_execution[b]) { pairs.push((*cost_a, *cost_b)); } @@ -260,6 +268,9 @@ pub(crate) fn copies_in_equivalent_contexts( .iter() .zip(selectable.edges_into(b, input)) { + if !kept[edges[edge_a].producer] { + continue; + } if let (Some(cost_a), Some(cost_b)) = (costs.per_edge[edge_a], costs.per_edge[edge_b]) { @@ -277,6 +288,9 @@ pub(crate) fn copies_in_equivalent_contexts( return pairs.iter().all(|&(a, b)| a == 0.0 && b == 0.0); } let alpha = largest_b / largest_a; + if alpha <= 0.0 { + return false; + } let tolerance = COST_RELATIVE_TOLERANCE * largest_b.abs(); pairs .iter() diff --git a/crates/stage3-plan-selection/src/milp/model/tests.rs b/crates/stage3-plan-selection/src/milp/model/tests.rs index 3d3566dc1..f5b68ae7b 100644 --- a/crates/stage3-plan-selection/src/milp/model/tests.rs +++ b/crates/stage3-plan-selection/src/milp/model/tests.rs @@ -30,7 +30,7 @@ use asap_types::workload::{ use super::*; use crate::cost::ExecutionCosts; use crate::plan::plan_of; -use crate::retention::{edge_retention_need_ms, least_need_ms_per_execution}; +use crate::retention::{edge_retention_need_ms, least_retention_ms_per_execution}; use crate::selectable::{QueryId, SelectableEdge}; const NONE: Lifetime = Lifetime::None; @@ -192,11 +192,12 @@ fn selection_inputs( .collect(), retention_cost_per_retained_second, newest_output_retention_ms: HashMap::new(), - least_need_ms_per_execution: Vec::new(), + least_retention_ms_per_execution: Vec::new(), raw_data_source: HashMap::new(), unbounded_raw_data_on_disk_cost_per_second: HashMap::new(), }; - costs.least_need_ms_per_execution = least_need_ms_per_execution(&selectable, &costs, entries); + costs.least_retention_ms_per_execution = + least_retention_ms_per_execution(&selectable, &costs, entries); (selectable, costs) } @@ -469,8 +470,8 @@ fn a_stored_result_read_at_its_newest_output_retains_one_output() { costs .newest_output_retention_ms .insert(PhysicalCandidateId(1), newest_output_ms); - costs.least_need_ms_per_execution = - least_need_ms_per_execution(&selectable, &costs, &entries); + costs.least_retention_ms_per_execution = + least_retention_ms_per_execution(&selectable, &costs, &entries); } solve_selection(&SelectionInputs { physical: &physical, @@ -1135,3 +1136,65 @@ fn contexts_whose_small_edge_costs_differ_by_rounding_are_copies() { assert_eq!(original != index, every_300s(index)); } } + +/// The same query `m[5m]` every 60 s and every 300 s over a scan with +/// ℓ = None (s0) or `OneEvaluation` (s1), read by the range (r) or by a +/// `OneEvaluation` range costing 1e12 (2e11 every 300 s), which the bound +/// leaves out. Every 60 s, r, s0 and s1 cost 1, 1 and 1.1; every 300 s +/// 0.2, 0.3 and 0.22, and the edge s1 → r 1e-3 (so s1 does not dominate +/// s0). The 300 s costs are not one multiple of the 60 s ones (s0 would +/// be 0.2), so the contexts are not copies, and the optimum reads s0 +/// every 60 s and s1 every 300 s: 2 + 0.421 = 2.421. Compared by every +/// cost, the left-out range's 2e11 sets the tolerance to 0.2, the +/// contexts share their variables, and the optimum is 2.5. +#[test] +fn a_left_out_execution_does_not_set_the_tolerance_of_context_copies() { + let mut memo = empty_memo(); + let m = scan(&mut memo); + let last_5m = range(&mut memo, 300, m); + query(&mut memo, last_5m, None); + query(&mut memo, last_5m, None); + let physical = physical( + memo, + &[ + (own_candidate(m), NONE), + (own_candidate(m), ONE_EVALUATION), + (own_candidate(last_5m), NONE), + (own_candidate(last_5m), ONE_EVALUATION), + ], + ); + let entries = [every(60_000), every(300_000)]; + let (selectable, mut costs) = + selection_inputs(&physical, &entries, &[1.0, 1.1, 1.0, 1e12], &[], &[]); + let executions = selectable.executions(); + let every_300s = |index: usize| { + matches!( + selectable.context(executions[index].context), + asap_types::stages::ExecutionContext::Evaluation { grid } if grid.period_ms == 300_000 + ) + }; + for (index, cost) in costs.per_execution.iter_mut().enumerate() { + if every_300s(index) { + *cost = Ok([0.3, 0.22, 0.2, 2e11][executions[index].candidate.0]); + } + } + let s1_to_range = edge(&physical, 1, 2); + for (position, selectable_edge) in selectable.edges().iter().enumerate() { + if selectable_edge.edge == s1_to_range && every_300s(selectable_edge.consumer) { + costs.per_edge[position] = Some(1e-3); + } + } + let solved = solve_selection(&SelectionInputs { + physical: &physical, + entries: &entries, + selectable: &selectable, + costs: &costs, + }) + .unwrap(); + assert!( + (solved.cost_per_second_without_request_overhead - 2.421).abs() < 1e-9, + "{}", + solved.cost_per_second_without_request_overhead + ); + assert_eq!(candidates(&solved), vec![0, 1, 2, 2]); +} diff --git a/crates/stage3-plan-selection/src/milp/retention.rs b/crates/stage3-plan-selection/src/milp/retention.rs index e3e9fed42..efc40d8b8 100644 --- a/crates/stage3-plan-selection/src/milp/retention.rs +++ b/crates/stage3-plan-selection/src/milp/retention.rs @@ -54,7 +54,7 @@ pub(crate) fn retention_variables( /// /// With them, per execution $u$ of such a $v$, $h_v \ge /// \min_{\text{uses}} \operatorname{need} \cdot x_u$ over its costed -/// reads (`ExecutionCosts::least_need_ms_per_execution`): (F) gives a +/// reads (`ExecutionCosts::least_retention_ms_per_execution`): (F) gives a /// selected execution a use, and (R) then retains at least the least /// need. It is implied for integer $x$, $y$, /// $z$, but tightens the relaxation where fractional $y$ leave $h_v$ low. @@ -83,7 +83,12 @@ pub(crate) fn retention_constraints( constraints.push(constraint!(h >= ms_to_seconds(need_ms) * read)); } } - for (index, need_ms) in inputs.costs.least_need_ms_per_execution.iter().enumerate() { + for (index, need_ms) in inputs + .costs + .least_retention_ms_per_execution + .iter() + .enumerate() + { if let (Some(&h), Some(need_ms)) = (h.get(&executions[index].candidate), *need_ms) { if need_ms > 0 { constraints.push(constraint!(h >= ms_to_seconds(need_ms) * x[index])); diff --git a/crates/stage3-plan-selection/src/milp/upper_bound.rs b/crates/stage3-plan-selection/src/milp/upper_bound.rs index 271395327..92da7901e 100644 --- a/crates/stage3-plan-selection/src/milp/upper_bound.rs +++ b/crates/stage3-plan-selection/src/milp/upper_bound.rs @@ -46,7 +46,7 @@ pub(crate) fn dearer_than_a_feasible_plan(inputs: &SelectionInputs<'_>) -> Vec { costs.retention_cost_per_second(v, need_ms) } diff --git a/crates/stage3-plan-selection/src/retention.rs b/crates/stage3-plan-selection/src/retention.rs index 7d1311fe6..15d8d3f3b 100644 --- a/crates/stage3-plan-selection/src/retention.rs +++ b/crates/stage3-plan-selection/src/retention.rs @@ -62,12 +62,12 @@ pub(crate) fn edge_scan_read_ms( /// bounded one. (F) gives a selected execution a use and (R) then retains /// at least that: the greedy bound charges it, and the MILP has /// $h_v \ge$ it $\cdot x$. -pub(crate) fn least_need_ms_per_execution( +pub(crate) fn least_retention_ms_per_execution( selectable: &SelectableExecutions, costs: &ExecutionCosts, entries: &[QueryWorkloadEntry], ) -> Vec> { - let mut least_need_ms = vec![None; selectable.executions().len()]; + let mut least_retention_ms = vec![None; selectable.executions().len()]; for (index, reader, need_ms) in selectable.reads_back(&costs.retention_need_ms_per_edge, entries) { @@ -76,13 +76,13 @@ pub(crate) fn least_need_ms_per_execution( Reader::Output(_) => true, }; if let (true, Some(need_ms)) = (costed, need_ms) { - let need_ms = + let retention_ms = costs.retention_ms_for_need(selectable.executions()[index].candidate, need_ms); - let least: &mut Option = &mut least_need_ms[index]; - *least = Some(least.map_or(need_ms, |least| least.min(need_ms))); + let least: &mut Option = &mut least_retention_ms[index]; + *least = Some(least.map_or(retention_ms, |least| least.min(retention_ms))); } } - least_need_ms + least_retention_ms } /// `ms` in seconds, the unit of $h_v$ in the MILP and of the costs per diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 66f29d0d0..49a361f31 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -1933,4 +1933,111 @@ mod tests { }; assert_eq!(operator.vector_match.as_ref(), Some(&vm)); } + + /// For every operator whose scalar expressions read operator nodes, + /// `map_children` calls `f` in [`NonASAPOp::children`] order (direct + /// inputs, then each node its scalar expressions read, as + /// `ScalarExpr::operator_refs` lists them), so a caller may hand out + /// replacements by position. Example: `Filter` over node 1 with + /// predicate `s(11) IN (SELECT … node 12) AND s(13)` sees 1, 12, 11, + /// 13: an `IN` lists its subquery before the nodes its left side reads. + #[test] + fn map_children_calls_f_in_children_order() { + let read = |id: u32| ScalarExpr::ScalarSubquery(id); + let in_and = |a: u32, b: u32, c: u32| { + ScalarExpr::BoolAnd(vec![ + ScalarExpr::InSubquery { + expr: Box::new(read(a)), + subquery: b, + negated: false, + }, + read(c), + ]) + }; + let key = |id: u32| SortKey { + expr: in_and(id, id + 1, id + 2), + ascending: true, + nulls_first: false, + }; + let empty_schema = Schema { + fields: vec![], + time_index: None, + unique_keys: vec![], + closed: true, + }; + let operators: Vec> = vec![ + NonASAPOp::Scan { + source: Source::Table { + table_ref: "t".into(), + }, + predicates: vec![Predicate(in_and(11, 12, 13)), Predicate(read(14))], + schema: empty_schema.clone(), + }, + NonASAPOp::Values { + rows: vec![vec![in_and(11, 12, 13)], vec![read(14)]], + schema: empty_schema, + }, + NonASAPOp::Filter { + pred: Predicate(in_and(11, 12, 13)), + child: 1, + }, + NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: None, + expr: in_and(11, 12, 13), + }, + ProjectItem { + alias: None, + expr: read(14), + }, + ], + child: 1, + }, + NonASAPOp::Aggregate { + reduction: Reduction::PerEntity, + measures: vec![], + output_names: vec![], + filters: vec![Some(Predicate(in_and(11, 12, 13))), None], + having: Some(Predicate(read(14))), + child: 1, + }, + NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(in_and(11, 12, 13)), + left: 1, + right: 2, + }, + NonASAPOp::Sort { + keys: vec![key(11), key(14)], + partition_by: GroupKeys::by(vec![]), + child: 1, + }, + NonASAPOp::SQLWindowFunc { + func: WindowFuncKind::RowNumber, + args: vec![in_and(11, 12, 13)], + partition_by: GroupKeys::by(vec![]), + order_by: vec![key(14)], + frame: None, + output_name: "w".into(), + child: 1, + }, + NonASAPOp::PromqlVectorFromScalar(in_and(11, 12, 13)), + NonASAPOp::PromqlRelabel { + dst: "d".into(), + value: in_and(11, 12, 13), + child: 1, + }, + ]; + for operator in &operators { + let mut seen = Vec::new(); + operator.map_children(|&id| { + seen.push(id); + id + }); + let children: Vec = operator.children().into_iter().copied().collect(); + assert_eq!(seen, children, "{operator:?}"); + } + assert_eq!(operators[2].children(), vec![&1, &12, &11, &13]); + } } diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index 234717316..fe128ffc4 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -263,11 +263,15 @@ impl ScalarExpr { expr, subquery, negated, - } => ScalarExpr::InSubquery { - expr: map_box(expr, column, f), - subquery: f(subquery), - negated: *negated, - }, + } => { + // The subquery first, as `operator_refs` lists it. + let subquery = f(subquery); + ScalarExpr::InSubquery { + expr: map_box(expr, column, f), + subquery, + negated: *negated, + } + } ScalarExpr::Negative { expr, semantics } => ScalarExpr::Negative { expr: map_box(expr, column, f), semantics: *semantics, diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index 77fe7eb35..7e91ce4b5 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -110,6 +110,20 @@ pub struct PhysicalASAPDAGNode { pub view_coverage: Option, } +impl PhysicalASAPDAGNode { + /// The nodes this node reads, one per edge into it in a validated DAG, + /// in its payload's input order ([`Operator::children`]): its direct + /// inputs, then each node its scalar expressions refer to, once. + /// Example: `q + scalar(count(q))` as a `Project` over `q` reads `q`, + /// then the count. + pub fn producers(&self) -> Vec { + edge_roles(&self.payload) + .into_iter() + .map(|(producer, _)| producer) + .collect() + } +} + /// A view's aggregate coverage /// ([`AggregateCoverage`](crate::stages::AggregateCoverage), `docs/design_docs/proposals/asap-primitive-schema.md` /// §4.2): what it computes and which rows went in.