diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index f811145d4..bd76af879 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -424,8 +424,8 @@ fn each_node_usage<'a>( let scan_window = match (&node.payload, node.output_state.timing) { (Payload::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { match dag.scan_window(node.id, kept_chains) { - // Priced as before #955 decides `@` anchors and - // unwindowed readers: the widest window read. + // An `@` anchor or an unwindowed reader: priced at + // the widest window read. ScanWindow::Unbounded => trailing_window( widest_window_read_ms(dag, nodes, node.id, kept_chains) .or(root_scan_lookback_ms), @@ -1617,11 +1617,12 @@ pub fn kept_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet, @@ -1647,7 +1648,7 @@ fn widest_window_read_ms( tumbling_window_count, .. }) => tumbling_window_width_ms.saturating_mul(u64::from(*tumbling_window_count)), - Payload::NonASAP(NonASAPOp::TimeShift { .. }) => { + Payload::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { stack.push(edge.consumer); continue; } diff --git a/crates/devtools/tests/memo_pricing.rs b/crates/devtools/tests/memo_pricing.rs index 1bdc9e850..582eae23d 100644 --- a/crates/devtools/tests/memo_pricing.rs +++ b/crates/devtools/tests/memo_pricing.rs @@ -339,12 +339,12 @@ fn scalar_root_choices_are_composed_and_priced() { assert!(priced.iter().any(|p| p.valid)); } -/// A choice whose selected plan installation refuses for a reason other -/// than the executor (here a raw-data window under a subquery, which no -/// runtime can place) is `not_installable`, never valid, so it is never -/// flagged cheaper than the selected plan. +/// A subquery's choices install: the executor reads a subquery's raw +/// samples at query time, so every choice of +/// `max_over_time(rate(data[5m])[1h:1m])` is valid, none cheaper than the +/// selected plan. #[test] -fn a_choice_installation_refuses_is_not_installable() { +fn subquery_choices_install() { let mut workload = ratio_of_quantiles(); workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query("max_over_time(rate(data[5m])[1h:1m])".into()); @@ -368,18 +368,6 @@ fn a_choice_installation_refuses_is_not_installable() { ); assert!(!priced.is_empty()); for p in &priced { - assert!(!p.valid && !p.cheaper_than_selected, "{p:?}"); - } - let refused: Vec<_> = priced - .iter() - .filter(|p| p.rejection.as_deref() == Some("not_installable")) - .collect(); - assert!(!refused.is_empty(), "{priced:?}"); - for p in refused { - let reason = p.rejection_reason.as_deref().unwrap(); - assert!( - reason.contains("a subquery or a TimeShift above the range"), - "{reason}" - ); + assert!(p.valid && !p.cheaper_than_selected, "{p:?}"); } } diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index eb391edbd..76a6ed9ec 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -111,9 +111,11 @@ enum Kind { }, Project(Vec), Filter(Expression), - /// A `TimeRange` over a `Scan` ([`Operator::time_range`]). + /// A `TimeRange` over a time-indexed input ([`Operator::time_range`]). TimeRange { time: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + at: Option, end_offset_ms: i64, width_ms: i64, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -238,8 +240,14 @@ impl Operator { } /// Set a counter evaluation's logical window: `lookback` long, ending - /// `offset` before the evaluation time. - pub(crate) fn with_counter_window(mut self, offset: i64, lookback: i64) -> Result { + /// `offset` before the evaluation time, or before the time of the `@` + /// anchor `at`. + pub(crate) fn with_counter_window( + mut self, + at: Option, + offset: i64, + lookback: i64, + ) -> Result { if lookback <= 0 { return Err(invalid("counter lookback must be positive")); } @@ -250,6 +258,7 @@ impl Operator { { evaluation.lookback_ms = Some(lookback); evaluation.offset_ms = offset; + evaluation.at = at; } Ok(self) } @@ -264,6 +273,7 @@ impl Operator { SummaryEvaluation::Exact(crate::summary_kernels::exact::ExactEvaluation { lookback_ms: Some(lookback), offset_ms, + at, .. }), .. @@ -271,11 +281,15 @@ impl Operator { else { return Ok(None); }; - let end = match context.scope { - crate::runtime::Scope::Query { - evaluation_time_ms, .. - } => evaluation_time_ms, - crate::runtime::Scope::Ingestion { window_end_ms, .. } => window_end_ms, + let end = match (at, &context.scope) { + (Some(at), _) => time_range::anchor_time(*at, context)?, + ( + None, + crate::runtime::Scope::Query { + evaluation_time_ms, .. + }, + ) => *evaluation_time_ms, + (None, crate::runtime::Scope::Ingestion { window_end_ms, .. }) => *window_end_ms, } .checked_sub(*offset_ms) .ok_or_else(|| invalid("counter window overflows Int64"))?; diff --git a/crates/executor/src/operators/series_window.rs b/crates/executor/src/operators/series_window.rs index 6d81f48df..4649f044b 100644 --- a/crates/executor/src/operators/series_window.rs +++ b/crates/executor/src/operators/series_window.rs @@ -138,16 +138,8 @@ pub(super) fn anchor( ) -> Result, Error> { match bound { None => Ok(literal), - Some(bound) => { - let (start, end) = context - .query_range() - .ok_or_else(|| invalid("@ start() and @ end() require query range bounds"))?; - match bound { - AtModifier::Start => Ok(Some(start)), - AtModifier::End => Ok(Some(end)), - AtModifier::Timestamp(_) => Err(invalid("invalid range-bound anchor")), - } - } + Some(AtModifier::Timestamp(_)) => Err(invalid("invalid range-bound anchor")), + Some(bound) => super::time_range::anchor_time(bound, context).map(Some), } } diff --git a/crates/executor/src/operators/time_range.rs b/crates/executor/src/operators/time_range.rs index 9e0540b62..af1f606f4 100644 --- a/crates/executor/src/operators/time_range.rs +++ b/crates/executor/src/operators/time_range.rs @@ -1,22 +1,26 @@ -//! A `TimeRange` over a `Scan`: the rows of one window on the time column, -//! for a SQL table and a time series alike. A time series' stale markers -//! (the Prometheus end-of-series NaN) are never rows; an ordinary NaN is. +//! A `TimeRange` over a time-indexed input (a `Scan`, or rows computed +//! from one): the rows of one window on the time column, for a SQL table +//! and a time series alike. A time series' stale markers (the Prometheus +//! end-of-series NaN) are never rows; an ordinary NaN is. use super::*; use arrow::array::Array; +use planner_types::ir::operator::AtModifier; impl Operator { /// The rows of `input` whose `time` lies in the window of a - /// `TimeRange(width_ms)` over shifts summing to `end_offset_ms`: `(T - - /// end_offset_ms - width_ms, T - end_offset_ms]` in a query run at `T`, - /// `(window_end_ms - width_ms, window_end_ms]` in an ingestion run. - /// With `sample_value`, a time series' value column, a stale marker is - /// not a row. With `latest_per_series` (an instant window), only each - /// series' latest row in the window is kept (of rows at one time, the - /// last), and none when that row is a stale marker; a series is the - /// values of those columns (its labels). + /// `TimeRange(width_ms)` over shifts: `(e - width_ms, e]`, with `e` the + /// window end [`window_end`] gives for the shifts' `@` anchor `at` and + /// offsets summing to `end_offset_ms` (`T - end_offset_ms` in a query + /// run at `T` without an anchor, the run's `window_end_ms` in an + /// ingestion run). With `sample_value`, a time series' value column, a + /// stale marker is not a row. With `latest_per_series` (an instant + /// window), only each series' latest row in the window is kept (of rows + /// at one time, the last), and none when that row is a stale marker; a + /// series is the values of those columns (its labels). pub fn time_range( input: SchemaRef, time: usize, + at: Option, end_offset_ms: i64, width_ms: i64, sample_value: Option, @@ -39,6 +43,7 @@ impl Operator { Ok(Self { kind: Kind::TimeRange { time, + at, end_offset_ms, width_ms, sample_value, @@ -50,24 +55,55 @@ impl Operator { } } -/// `(start, end]` of a window `width_ms` wide in this run: ending -/// `end_offset_ms` before the evaluation time `T` in a query run; ending at -/// the run's `window_end_ms` in an ingestion run, which builds the window -/// the shifts select (the shifts say which window a query reads, the -/// ingestion run's window which one it builds). +/// The time an `@` anchor pins: `@ ` its instant, `@ start()` and +/// `@ end()` the query range's bounds, which an instant query (a run +/// without them) has both at its evaluation time, as in Prometheus. An +/// ingestion run has no anchor time: an anchored window is read at query +/// time. +pub(super) fn anchor_time(at: AtModifier, context: &RunContext) -> Result { + let crate::runtime::Scope::Query { evaluation_time_ms } = context.scope else { + return Err(invalid("an @-anchored window is read only at query time")); + }; + let (start, end) = context + .query_range() + .unwrap_or((evaluation_time_ms, evaluation_time_ms)); + Ok(match at { + AtModifier::Timestamp(ms) => ms, + AtModifier::Start => start, + AtModifier::End => end, + }) +} + +/// The end of a window `end_offset_ms` before its reference time: the time +/// of the `@` anchor `at` ([`anchor_time`]) or, without one, the evaluation +/// time `T` in a query run; in an ingestion run, the run's `window_end_ms`, +/// which builds the window the shifts select (the shifts say which window +/// a query reads, the ingestion run's window which one it builds). +pub(super) fn window_end( + at: Option, + end_offset_ms: i64, + context: &RunContext, +) -> Result { + let reference = match (at, &context.scope) { + (Some(at), _) => anchor_time(at, context)?, + (None, crate::runtime::Scope::Query { evaluation_time_ms }) => *evaluation_time_ms, + (None, crate::runtime::Scope::Ingestion { window_end_ms, .. }) => { + return Ok(*window_end_ms) + } + }; + reference + .checked_sub(end_offset_ms) + .ok_or_else(|| invalid("time range overflows Int64")) +} + +/// `(start, end]` of a window `width_ms` wide ending at [`window_end`]. pub(super) fn window( + at: Option, end_offset_ms: i64, width_ms: i64, context: &RunContext, ) -> Result<(i64, i64), Error> { - let end = match context.scope { - crate::runtime::Scope::Query { - evaluation_time_ms, .. - } => evaluation_time_ms - .checked_sub(end_offset_ms) - .ok_or_else(|| invalid("time range overflows Int64"))?, - crate::runtime::Scope::Ingestion { window_end_ms, .. } => window_end_ms, - }; + let end = window_end(at, end_offset_ms, context)?; let start = end .checked_sub(width_ms) .ok_or_else(|| invalid("time range overflows Int64"))?; @@ -81,6 +117,7 @@ pub(super) fn execute<'a>( ) -> Result, Error> { let Kind::TimeRange { time, + at, end_offset_ms, width_ms, sample_value, @@ -92,7 +129,7 @@ pub(super) fn execute<'a>( let input = inputs .pop() .ok_or_else(|| invalid("time range input missing"))?; - let (start, end) = window(end_offset_ms, width_ms, &context)?; + let (start, end) = window(at, end_offset_ms, width_ms, &context)?; // Whether each row is in the window and, for a range window, not a // stale marker. let kept = diff --git a/crates/executor/src/operators/unchecked.rs b/crates/executor/src/operators/unchecked.rs index 173dce2d1..7f3449ecd 100644 --- a/crates/executor/src/operators/unchecked.rs +++ b/crates/executor/src/operators/unchecked.rs @@ -110,6 +110,7 @@ impl TryFrom for Operator { Kind::Filter(expression) => Operator::filter(input(0)?, expression)?, Kind::TimeRange { time, + at, end_offset_ms, width_ms, sample_value, @@ -117,6 +118,7 @@ impl TryFrom for Operator { } => Operator::time_range( input(0)?, time, + at, end_offset_ms, width_ms, sample_value, diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index c78dec714..3a06484af 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -12,10 +12,11 @@ use crate::{ }; use planner_types::ir::materialized_view::MaterializedView; use planner_types::ir::operator::{ - agg_has_exact_per_entity_kernel, AggIntent, GroupKeys, Reduction as PlannerReduction, + agg_has_exact_per_entity_kernel, AggIntent, AtModifier, GroupKeys, + Reduction as PlannerReduction, }; use planner_types::ir::physical_export::{ - time_range_window, PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, + PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, ScanWindow, }; use planner_types::ir::scalar::{ColumnRef, CompareOpKind}; @@ -143,6 +144,7 @@ fn compile_internal( } } let mut ordered = Vec::new(); + let mut subqueries = BTreeMap::new(); let mut seen = BTreeSet::new(); let mut pending = roots.iter().map(|&id| (id, false)).collect::>(); while let Some((id, expanded)) = pending.pop() { @@ -158,6 +160,11 @@ fn compile_internal( } pending.push((id, true)); if !views_by_node.contains_key(&id) { + // A subquery reads its selector's `Scan` directly. + if let Some(subquery) = subquery(id, &nodes, &dependencies)? { + dependencies.insert(id, vec![subquery.scan]); + subqueries.insert(id, subquery); + } for &input in dependencies.get(&id).into_iter().flatten() { pending.push((input, false)); } @@ -221,9 +228,13 @@ fn compile_internal( .map_err(|error| invalid(format!("scan {id}: {error}")))?; // The rows its readers read; a compile root reads them all. let window = match output.time_index { - Some(_) if !compile_roots.contains(&id) => { - scan_windows[&(id as PhysicalASAPNodeId)] - } + Some(_) if !compile_roots.contains(&id) => subqueries + .values() + .filter(|subquery: &&Subquery| subquery.scan == id) + .fold( + scan_windows[&(id as PhysicalASAPNodeId)], + |window, subquery| window_hull(window, subquery.scan_window()), + ), _ => ScanWindow::Unbounded, }; physical_dag.add_raw_data( @@ -465,15 +476,23 @@ fn compile_internal( let [input_id] = inputs.as_slice() else { return Err(invalid("per-entity summary requires one input")); }; - let Some(NonASAPOp::TimeRange { child, .. }) = restored[input_id].non_asap() else { - return Err(invalid( - "per-entity summary requires a resolved raw time range", - )); + // The schema of the source whose series it summarizes: a + // subquery's, or the one a raw time range reads (a tumbling + // window reads its window through `TimeShift`s; the + // deployment supplies the shifted raw rows). + let schema = match (subqueries.get(input_id), restored[input_id].non_asap()) { + (Some(subquery), _) => match &nodes[&subquery.scan].payload { + Payload::NonASAP(NonASAPOp::Scan { schema, .. }) => Some(schema), + _ => None, + }, + (None, Some(NonASAPOp::TimeRange { child, .. })) => scan_schema_below(child), + _ => { + return Err(invalid( + "per-entity summary requires a resolved raw time range", + )) + } }; - // A tumbling window reads its window through `TimeShift`s; the - // deployment supplies the shifted raw rows. - let source = below_time_shifts(child, |node| node.non_asap(), Ok)?; - let Some(NonASAPOp::Scan { schema, .. }) = source.non_asap() else { + let Some(schema) = schema else { return Err(invalid("per-entity summary requires a resolved source")); }; // An unkeyed frequency family (UnivMon) counts each value of @@ -631,6 +650,18 @@ fn compile_internal( continue; } } + if let Some(subquery) = subqueries.get(&id) { + let mut operators = subquery + .operators(schemas[0].clone(), output) + .map_err(|error| invalid(format!("node {id}: {error}")))?; + let last = operators.pop().expect("a subquery runs a series window"); + if let Some(window) = operators.pop() { + physical_dag.add(auxiliary, inputs, window)?; + inputs = vec![auxiliary]; + } + physical_dag.add(id, inputs, last)?; + continue; + } // A window over a table or a time series (and a tumbling window's // shift under it) runs natively on the time column. if let Some(operator) = time_window_operator(id, &nodes, &dependencies, &schemas) @@ -642,8 +673,10 @@ fn compile_internal( let mut operator = compile_node(node, &schemas) .map_err(|error| invalid(format!("node {id}: {error}")))?; if operator.is_counter_evaluation() { - if let Some((offset, lookback)) = counter_window(id, &nodes, &dependencies)? { - operator = operator.with_counter_window(offset, lookback)?; + if let Some((at, offset, lookback)) = + counter_window(id, &nodes, &dependencies, &subqueries)? + { + operator = operator.with_counter_window(at, offset, lookback)?; } } if temporal_evaluation_drops_name(node) { @@ -675,70 +708,74 @@ fn built_from_relation(state: &OperatorNode) -> bool { } } -/// The native operator of `id` when it is a `TimeRange` or a `TimeShift` -/// over a `Scan` (below any stacked `TimeShift`s), of a SQL table or a time -/// series; `None` for any other node. +/// The native operator of `id` when it is a `TimeRange` over any +/// time-indexed input (a `Scan` of a SQL table or a time series, or rows +/// computed from one, below any stacked `TimeShift`s), or a `TimeShift` +/// that is not a subquery's ([`Subquery`]); `None` for any other node. /// -/// A `TimeRange(w)` over shifts summing to `s` keeps the rows whose time -/// column lies in `(T - s - w, T - s]` at evaluation time `T` -/// ([`time_range_window`]), the window a PromQL selector -/// and a SQL filter against `now()` read; an ingestion run builds the -/// window ending at its `window_end_ms` ([`Operator::time_range`]). Over a -/// time series a stale marker is not a row, and an instant window keeps -/// each series' (each label set's) latest row. A `TimeShift` only moves -/// the window its `TimeRange` reads (the range above applies its offset), -/// so it passes the rows through; a shift is meaningful only under a -/// range, the shape the frontends and Pass 2's tumbling windows build. An -/// `@` anchor has no lowering, so a range or shift over one gets `None` and -/// the node fails compilation; an instant window over a table is refused. +/// A `TimeRange(w)` keeps the rows whose time column lies in `(e - w, e]`, +/// with `e` the end its shifts give ([`shifted_end`]): `T - s` at +/// evaluation time `T` for offsets summing to `s`, the window a PromQL +/// selector and a SQL filter against `now()` read; an `@` anchor's time +/// minus the offsets from the anchored shift down; an ingestion run builds +/// the window ending at its `window_end_ms` ([`Operator::time_range`]). +/// Over a time series a stale marker is not a row, and an instant window +/// keeps each series' (each label set's) latest row. A `TimeShift` only +/// moves the window its `TimeRange` reads (the range above applies it), so +/// over a `Scan`'s rows it passes them through: a shift is meaningful only +/// under a range, the shape the frontends and Pass 2's tumbling windows +/// build; over rows evaluated at a time (a range's, an aggregate's) it is +/// refused. An instant window over a table is refused. fn time_window_operator( id: NodeId, nodes: &BTreeMap, dependencies: &BTreeMap>, input_schemas: &[SchemaRef], ) -> Result, Error> { - let below = match &nodes[&id].payload { - Payload::NonASAP(NonASAPOp::TimeRange { .. }) => first_input(id, dependencies), - Payload::NonASAP(NonASAPOp::TimeShift { .. }) => Some(id), + let (range, kind) = match &nodes[&id].payload { + Payload::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => (range, kind), + Payload::NonASAP(NonASAPOp::TimeShift { .. }) => { + let [input] = input_schemas else { + return Err(invalid("a time shift reads one input")); + }; + // Rows evaluated at another time (a range's, an aggregate's) + // cannot be moved by passing them through. + let (_, _, below) = shifted_end(Some(id), nodes, dependencies)?; + if !reads_scan_rows(below, nodes, dependencies) { + return Err(invalid( + "a time shift over an evaluated input has no native implementation", + )); + } + let columns = input + .fields + .iter() + .enumerate() + .map(|(i, field)| (field.name.clone(), Expression::Column(i))) + .collect(); + return Operator::project(input.clone(), columns).map(Some); + } _ => return Ok(None), }; - let Some(below) = node_below_shifts(below, nodes, dependencies) else { - return Ok(None); - }; - let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &nodes[&below].payload else { - return Ok(None); - }; let [input] = input_schemas else { - return Err(invalid("a time range or shift over a Scan reads one input")); - }; - let Payload::NonASAP(NonASAPOp::TimeRange { kind, .. }) = &nodes[&id].payload else { - // A shift passes its rows through. - let columns = input - .fields - .iter() - .enumerate() - .map(|(i, field)| (field.name.clone(), Expression::Column(i))) - .collect(); - return Operator::project(input.clone(), columns).map(Some); - }; - let ScanWindow::Within { - end_offset_ms, - width_ms, - } = time_range_window(id as PhysicalASAPNodeId, |id| { - nodes.get(&(id as NodeId)).map(|node| &node.payload) - }) - else { - return Ok(None); + return Err(invalid("a time range reads one input")); }; + let (at, end_offset_ms, below) = + shifted_end(first_input(id, dependencies), nodes, dependencies)?; + let width_ms = + i64::try_from(range.as_millis()).map_err(|_| invalid("time range exceeds Int64"))?; let time = input .time_index - .ok_or_else(|| invalid("a time range needs its source's time column"))?; - let (sample_value, latest_per_series) = match (source, kind) { - (planner_types::ir::operator::Source::Table { .. }, TimeRangeKind::Range) => (None, None), - (planner_types::ir::operator::Source::Table { .. }, TimeRangeKind::Instant) => { + .ok_or_else(|| invalid("a time range needs its input's time column"))?; + let time_series = matches!( + source_below(below, nodes, dependencies), + Some(planner_types::ir::operator::Source::TimeSeries { .. }) + ); + let (sample_value, latest_per_series) = match (time_series, kind) { + (false, TimeRangeKind::Range) => (None, None), + (false, TimeRangeKind::Instant) => { return Err(invalid("an instant window over a SQL table is unsupported")) } - (planner_types::ir::operator::Source::TimeSeries { .. }, kind) => { + (true, kind) => { let value = named_column(input, &ColumnRef::SampleValue)?; // A series is its label set: every column but its time and value. let series = (*kind == TimeRangeKind::Instant).then(|| { @@ -752,6 +789,7 @@ fn time_window_operator( Operator::time_range( input.clone(), time, + at, end_offset_ms, width_ms, sample_value, @@ -760,90 +798,393 @@ fn time_window_operator( .map(Some) } +/// The smallest window holding both. +fn window_hull(a: ScanWindow, b: ScanWindow) -> ScanWindow { + let ( + ScanWindow::Within { + end_offset_ms: a_end, + width_ms: a_width, + }, + ScanWindow::Within { + end_offset_ms: b_end, + width_ms: b_width, + }, + ) = (a, b) + else { + return ScanWindow::Unbounded; + }; + let end_offset_ms = a_end.min(b_end); + let start_back_ms = a_end + .saturating_add(a_width) + .max(b_end.saturating_add(b_width)); + ScanWindow::Within { + end_offset_ms, + width_ms: start_back_ms.saturating_sub(end_offset_ms), + } +} + fn first_input(id: NodeId, dependencies: &BTreeMap>) -> Option { dependencies.get(&id).and_then(|d| d.first()).copied() } -/// The first node at or below `start` that is not a `TimeShift`; `None` -/// when `start` is `None`, a shift has an `@` anchor, or a shift lacks its -/// child. -fn node_below_shifts( +/// The end of the window read through the `TimeShift`s from `start` down, +/// as `(at, end_offset_ms)`: `end_offset_ms` before the time of the `@` +/// anchor `at`, or before the evaluation time without one; and the first +/// node below them. A shift evaluates its input at its anchor's time (or +/// the time it is evaluated at) minus its offset, so the deepest anchor +/// wins and only the offsets from it down count, as in Prometheus. An +/// error when `start` is `None` or a shift lacks its input. +fn shifted_end( start: Option, nodes: &BTreeMap, dependencies: &BTreeMap>, -) -> Option { +) -> Result<(Option, i64, NodeId), Error> { + let (mut at, mut end_offset_ms) = (None, 0i64); let mut below = start; while let Some(node) = below { let Payload::NonASAP(NonASAPOp::TimeShift { shift, .. }) = &nodes[&node].payload else { - return Some(node); + return Ok((at, end_offset_ms, node)); }; if shift.at.is_some() { - return None; + (at, end_offset_ms) = (shift.at, shift.offset_ms); + } else { + end_offset_ms = end_offset_ms + .checked_add(shift.offset_ms) + .ok_or_else(|| invalid("time shift overflows Int64"))?; + } + below = first_input(node, dependencies); + } + Err(invalid("a time range or shift lacks its input")) +} + +/// Whether `node`'s rows are a `Scan`'s, through row-wise `Project`s and +/// `Filter`s. +fn reads_scan_rows( + node: NodeId, + nodes: &BTreeMap, + dependencies: &BTreeMap>, +) -> bool { + let mut below = Some(node); + while let Some(node) = below { + match &nodes[&node].payload { + Payload::NonASAP(NonASAPOp::Scan { .. }) => return true, + Payload::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => { + below = first_input(node, dependencies) + } + _ => return false, + } + } + false +} + +/// The source of the `Scan` reached from `node` down its first inputs. +fn source_below<'a>( + node: NodeId, + nodes: &BTreeMap, + dependencies: &BTreeMap>, +) -> Option<&'a planner_types::ir::operator::Source> { + let mut below = Some(node); + while let Some(node) = below { + if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &nodes[&node].payload { + return Some(source); } below = first_input(node, dependencies); } None } -/// The first node at or below `node` that is not a `TimeShift`, following -/// each shift's child: a range reads its raw source through any number of -/// stacked shifts. `operator` gives a node's non-ASAP operator (`None` for an -/// ASAP one) and `resolve` a child's node; an error when a child is missing. -pub(crate) fn below_time_shifts<'a, N, C: 'a>( - mut node: &'a N, - operator: impl Fn(&'a N) -> Option<&'a NonASAPOp>, - resolve: impl Fn(&'a C) -> Result<&'a N, Error>, -) -> Result<&'a N, Error> { - while let Some(NonASAPOp::TimeShift { child, .. }) = operator(node) { - node = resolve(child)?; +/// The default step of a subquery written without one (`[10m:]`): +/// Prometheus' default global evaluation interval, one minute. +const DEFAULT_SUBQUERY_STEP_MS: i64 = 60_000; + +/// A PromQL subquery the executor runs natively: `[R:s]`, under +/// the `TimeShift`s of its node (none when the node is the +/// `PromqlSubquery` itself), evaluates `inner` at every multiple of `s` in +/// `(e - R, e]`, with `e` the end those shifts give ([`shifted_end`]). +/// `inner` is an instant selector (`m`) or a per-series range function +/// (`f(m[r])`, an exact per-entity `Aggregate` over a range), over the +/// `Scan` `scan` below its own shifts; it runs as one +/// [`Operator::series_window`] over that `Scan`'s rows. +struct Subquery { + scan: NodeId, + /// `None` for an instant selector. + function: Option>, + /// The selector's range, or an instant selector's lookback. + range_ms: i64, + at: Option, + offset_ms: i64, + /// The step grid, without its anchor. + steps: crate::operators::SubquerySteps, + /// The anchor of the step grid's end. + steps_at: Option, +} + +/// The [`Subquery`] node `id` is: a `PromqlSubquery`, or a `TimeShift` +/// over one; `None` for any other node. An error for a subquery over an +/// expression other than a selector or a per-series range function of one. +fn subquery( + id: NodeId, + nodes: &BTreeMap, + dependencies: &BTreeMap>, +) -> Result, Error> { + if !matches!( + nodes[&id].payload, + Payload::NonASAP(NonASAPOp::TimeShift { .. } | NonASAPOp::PromqlSubquery { .. }) + ) { + return Ok(None); + } + let (steps_at, steps_offset_ms, below) = shifted_end(Some(id), nodes, dependencies)?; + let Payload::NonASAP(NonASAPOp::PromqlSubquery { + range: subquery_range, + resolution, + .. + }) = &nodes[&below].payload + else { + return Ok(None); + }; + let millis = |duration: &std::time::Duration| { + i64::try_from(duration.as_millis()).map_err(|_| invalid("subquery range exceeds Int64")) + }; + let unsupported = || { + invalid(format!( + "node {id}: a subquery over an expression other than a selector or a per-series \ + range function of one has no native implementation" + )) + }; + let inner = first_input(below, dependencies).ok_or_else(unsupported)?; + let (function, range) = match &nodes[&inner].payload { + Payload::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Instant, + .. + }) => (None, (inner, range)), + Payload::NonASAP(NonASAPOp::Aggregate { + reduction: PlannerReduction::PerEntity, + measures, + filters, + having: None, + .. + }) if filters.iter().all(Option::is_none) => { + let [measure] = measures.as_slice() else { + return Err(unsupported()); + }; + let window = first_input(inner, dependencies).ok_or_else(unsupported)?; + let Payload::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + .. + }) = &nodes[&window].payload + else { + return Err(unsupported()); + }; + // A range function of one series' samples names no column, so + // its intent carries over to the series window's column type. + let function = serde_json::to_value(measure) + .and_then(serde_json::from_value) + .map_err(|_| unsupported())?; + (Some(function), (window, range)) + } + // The exact accumulator of a range function, finalized. + Payload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) => { + let build = first_input(inner, dependencies).ok_or_else(unsupported)?; + let window = first_input(build, dependencies).ok_or_else(unsupported)?; + let ( + Payload::ASAP(ASAPOp::SummaryAgg { + family: FieldDataType::ExactAggregate(kind, _), + reduction: PlannerReduction::PerEntity, + filter: None, + .. + }), + Payload::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + .. + }), + ) = (&nodes[&build].payload, &nodes[&window].payload) + else { + return Err(unsupported()); + }; + use planner_types::ir::schema::ExactKind; + let function = match kind { + ExactKind::Sum => AggIntent::Sum { col: None }, + ExactKind::Count => AggIntent::Count { + accuracy: planner_types::ir::properties::AccuracyTarget::exact(), + }, + ExactKind::Min => AggIntent::Min { col: None }, + ExactKind::Max => AggIntent::Max { col: None }, + ExactKind::Increase => AggIntent::Increase, + ExactKind::Rate => AggIntent::Rate, + ExactKind::IRate => AggIntent::IRate, + }; + (Some(function), (window, range)) + } + _ => return Err(unsupported()), + }; + let (at, offset_ms, scan) = + shifted_end(first_input(range.0, dependencies), nodes, dependencies)?; + if !matches!( + nodes[&scan].payload, + Payload::NonASAP(NonASAPOp::Scan { .. }) + ) { + return Err(unsupported()); + } + let steps = crate::operators::SubquerySteps { + range_ms: millis(subquery_range)?, + step_ms: resolution + .as_ref() + .map(millis) + .transpose()? + .unwrap_or(DEFAULT_SUBQUERY_STEP_MS), + offset_ms: steps_offset_ms, + at_ms: None, + }; + Ok(Some(Subquery { + scan, + function, + range_ms: millis(range.1)?, + at, + offset_ms, + steps, + steps_at, + })) +} + +impl Subquery { + /// `(start, end]` back from the evaluation time of the rows it reads; + /// `Unbounded` with an `@` anchor. + fn scan_window(&self) -> ScanWindow { + if self.at.is_some() || self.steps_at.is_some() { + return ScanWindow::Unbounded; + } + ScanWindow::Within { + end_offset_ms: self.steps.offset_ms.saturating_add(self.offset_ms), + width_ms: self.steps.range_ms.saturating_add(self.range_ms), + } + } + + /// The operators that run it over the `Scan`'s rows `input`, giving + /// `output`: the series window, then, for a range function that drops + /// the metric name (all but `last_over_time`), the name dropped from + /// each series identity. + fn operators(&self, input: SchemaRef, output: SchemaRef) -> Result, Error> { + // An `@ ` is a literal; `@ start()` and `@ end()` are resolved + // at run time. + let pinned = |at: Option| match at { + Some(AtModifier::Timestamp(ms)) => (Some(ms), None), + bound => (None, bound), + }; + let ((at_ms, range_at), (steps_at_ms, steps_range_at)) = + (pinned(self.at), pinned(self.steps_at)); + let window = Operator::series_window( + input, + self.function.clone(), + self.range_ms, + self.offset_ms, + at_ms, + Some(crate::operators::SubquerySteps { + at_ms: steps_at_ms, + ..self.steps + }), + )? + .with_series_range_bounds(range_at, steps_range_at)?; + let drops_name = self + .function + .as_ref() + .is_some_and(|function| !matches!(function, AggIntent::LastOverTime)) + && output + .fields + .iter() + .any(|field| field.name == promql_rows::SERIES_IDENTITY_COLUMN); + Ok(if drops_name { + let windowed = window.schema(); + vec![ + window, + Operator::series_without_name(windowed)?.with_output_schema(output)?, + ] + } else { + vec![window.with_output_schema(output)?] + }) + } +} + +/// The schema of the `Scan` a range over `node` reads: `node`, or the +/// first node below it past `TimeShift`s and row-wise `Project`s and +/// `Filter`s (a SQL `asap_rate` projects its series' rows); `None` when +/// that is no `Scan`. +fn scan_schema_below(mut node: &OperatorNode) -> Option<&planner_types::ir::schema::Schema> { + loop { + match node.non_asap()? { + NonASAPOp::Scan { schema, .. } => return Some(schema), + NonASAPOp::TimeShift { child, .. } + | NonASAPOp::Project { child, .. } + | NonASAPOp::Filter { child, .. } => node = child, + _ => return None, + } } - Ok(node) } -/// The logical window of the counter evaluation `id`, as `(offset, -/// lookback)`: it ends `offset` ms before the evaluation time and spans -/// `lookback` ms. It is the union of the range windows the evaluation -/// reads, each moved back by the time shifts below its range: one range for -/// a single build, back-to-back tumbling windows for merged tumbling windows. `None` +/// The logical window of the counter evaluation `id`, as `(at, offset, +/// lookback)`: it ends `offset` ms before the evaluation time, or before +/// the time of the `@` anchor `at`, and spans `lookback` ms. It is the +/// union of the range windows the evaluation reads, each moved by the time +/// shifts below its range ([`shifted_end`]), a subquery's its step grid's +/// (`rate(x[10m:1m])`): one range for a single build, +/// back-to-back tumbling windows for merged tumbling windows. `None` /// without a range; an error when the ranges do not form one window. fn counter_window( id: NodeId, nodes: &BTreeMap, dependencies: &BTreeMap>, -) -> Result, Error> { + subqueries: &BTreeMap, +) -> Result, i64, i64)>, Error> { let millis = |ms: u128| i64::try_from(ms).map_err(|_| invalid("counter lookback exceeds Int64")); let mut pending = vec![id]; let mut visited = BTreeSet::new(); - // (offset, end of the window back from the evaluation time) per range. + // (offset, end of the window back from its reference time) per range, + // and the ranges' one anchor. let mut windows = BTreeSet::new(); + let mut anchors = Vec::new(); while let Some(ancestor) = pending.pop() { if !visited.insert(ancestor) { continue; } - if let Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) = &nodes[&ancestor].payload { - let mut offset = 0i64; - let mut below = dependencies.get(&ancestor).and_then(|d| d.first()).copied(); - while let Some(input) = below { - let Payload::NonASAP(NonASAPOp::TimeShift { shift, .. }) = &nodes[&input].payload - else { - break; - }; - if shift.at.is_none() { - offset = offset - .checked_add(shift.offset_ms) - .ok_or_else(|| invalid("counter window overflows Int64"))?; - } - below = dependencies.get(&input).and_then(|d| d.first()).copied(); + // A subquery is the range its outer function reads. + if let Some(subquery) = subqueries.get(&ancestor) { + let steps = subquery.steps; + let back = steps + .offset_ms + .checked_add(steps.range_ms) + .ok_or_else(|| invalid("counter window overflows Int64"))?; + windows.insert((steps.offset_ms, back)); + if !anchors.contains(&subquery.steps_at) { + anchors.push(subquery.steps_at); } + continue; + } + if let Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) = &nodes[&ancestor].payload { + let (at, offset, _) = + shifted_end(first_input(ancestor, dependencies), nodes, dependencies)?; let back = offset .checked_add(millis(range.as_millis())?) .ok_or_else(|| invalid("counter window overflows Int64"))?; windows.insert((offset, back)); + if !anchors.contains(&at) { + anchors.push(at); + } continue; } pending.extend(dependencies.get(&ancestor).into_iter().flatten().copied()); } + let at = match anchors.as_slice() { + [] => return Ok(None), + [at] => *at, + _ => { + return Err(invalid( + "counter evaluation reads windows of different anchors", + )) + } + }; let mut windows = windows.into_iter(); let Some((offset, mut back)) = windows.next() else { return Ok(None); @@ -855,7 +1196,7 @@ fn counter_window( } back = end; } - Ok(Some((offset, back - offset))) + Ok(Some((at, offset, back - offset))) } // Temporal summary evaluations produce PromQL vectors, whose range functions drop @@ -1024,6 +1365,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result for ExactAccumulator { } /// Planned evaluation of an exact summary. `lookback_ms` is the logical PromQL -/// counter window, ending `offset_ms` before the evaluation time (PromQL -/// `offset`); the evaluation range is resolved from them at run time. +/// counter window, ending `offset_ms` (PromQL `offset`) before the +/// evaluation time, or before the time of the `@` anchor `at`; the +/// evaluation range is resolved from them at run time. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub struct ExactEvaluation { pub statistic: ExactKind, @@ -78,6 +79,8 @@ pub struct ExactEvaluation { pub lookback_ms: Option, #[serde(default, skip_serializing_if = "is_zero")] pub offset_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub at: Option, } fn is_zero(ms: &i64) -> bool { diff --git a/crates/executor/tests/exact_state_columns.rs b/crates/executor/tests/exact_state_columns.rs index 27cf337b7..87143bf16 100644 --- a/crates/executor/tests/exact_state_columns.rs +++ b/crates/executor/tests/exact_state_columns.rs @@ -120,6 +120,7 @@ fn exact_records_merge_and_evaluate_as_kernel_objects_do() { statistic: kind, lookback_ms: None, offset_ms: 0, + at: None, }), ) .unwrap(); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index bd22b97d5..42478f88b 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -189,6 +189,7 @@ fn summary_construction_merge_and_evaluation_at_both_phases() { statistic: ExactKind::Sum, lookback_ms: None, offset_ms: 0, + at: None, }, ), ) @@ -507,6 +508,7 @@ fn exact_state_and_family_validation() { statistic: ExactKind::Sum, lookback_ms: None, offset_ms: 0, + at: None, }, ), ) @@ -631,6 +633,7 @@ fn empty_exact_count_is_an_integer_state_evaluation() { statistic: ExactKind::Count, lookback_ms: None, offset_ms: 0, + at: None, }, ), ) diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 2831befe1..df4993d05 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -865,6 +865,7 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { statistic: kind, lookback_ms: None, offset_ms: 0, + at: None, }, ), ) diff --git a/crates/executor/tests/promql_values.rs b/crates/executor/tests/promql_values.rs index 0585c4198..7e052a028 100644 --- a/crates/executor/tests/promql_values.rs +++ b/crates/executor/tests/promql_values.rs @@ -219,6 +219,7 @@ fn compile_exact_evaluation( statistic, lookback_ms: is_counter.then_some(lookback_ms), offset_ms: 0, + at: None, }), )?; let project = Operator::project( diff --git a/crates/integration-tests/tests/runtime_handoff.rs b/crates/integration-tests/tests/runtime_handoff.rs index cadfe8788..d4d2e05a3 100644 --- a/crates/integration-tests/tests/runtime_handoff.rs +++ b/crates/integration-tests/tests/runtime_handoff.rs @@ -275,53 +275,28 @@ async fn installable_at_query_time(query: &str) -> Result assert_eq!(refused, reason, "{query}"), - Err(error) => { - // When planning fails before installation, its no-plan - // diagnostic must keep the concrete refusal reason. - let diagnostic = error.to_string(); - assert!(diagnostic.contains(reason), "{query}: {diagnostic}"); - assert!(diagnostic.contains("raw-data window"), "{diagnostic}"); - } - Ok(plan) => panic!( - "{query}: unexpectedly installed {:?}", - plan.raw_data_retention().unwrap() - ), - } + let plan = installable_at_query_time(query) + .await + .unwrap_or_else(|error| panic!("{query}: {error}")); + let [retention] = plan.raw_data_retention().unwrap().try_into().unwrap(); + assert_eq!(retention.lookback_ms(), lookback_ms, "{query}"); } - let plain = installable_at_query_time("max_over_time(latency_ms[10m] offset 5m)") - .await - .unwrap(); - // 10 minutes of data ending 5 minutes back reaches 15 minutes back. - let [retention] = plain.raw_data_retention().unwrap().try_into().unwrap(); - assert_eq!(retention.lookback_ms(), Some(900_000)); } /// Example 4b answered as the last hour at 10-minute steps (a range diff --git a/crates/integration-tests/tests/tumbling_window_merge.rs b/crates/integration-tests/tests/tumbling_window_merge.rs index 7f5c7cff2..3246b5cb6 100644 --- a/crates/integration-tests/tests/tumbling_window_merge.rs +++ b/crates/integration-tests/tests/tumbling_window_merge.rs @@ -502,10 +502,8 @@ fn offset_counter_window_ends_at_the_offset() { /// A counter over `[5m] @ 300` reads the window ending at 300 s whatever the /// evaluation time, and `@ 300 offset 1m` the one ending at 240 s: the same -/// values as the unanchored counter evaluated at 300 s and at 240 s. Today -/// it reads the window ending at the evaluation time (#748). +/// values as the unanchored counter evaluated at 300 s and at 240 s. #[test] -#[ignore = "the counter window drops an @ anchor and its offset: #748"] fn anchored_counter_window_ends_at_the_anchor() { let samples = samples(7_000); for function in ["rate", "increase"] { diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index ea836d674..1078b31cd 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -859,19 +859,24 @@ mod tests { ); } - /// `TimeShift(10 min) <- TimeRange(5 min) <- Scan`, all at query time: - /// the shift above the range moves its window, which a window of a - /// fixed offset below the range cannot say, so it is refused; without - /// the shift the window is listed. + /// `TimeShift(10 min) <- TimeRange(5 min) <- Scan`: the shift above the + /// range moves its window, which a window of a fixed offset below the + /// range cannot say, so a view refreshed from it is refused (a query + /// run checks no raw-data window shape); without the shift the window + /// is listed. #[test] - fn a_time_shift_above_the_range_is_refused() { + fn a_time_shift_above_the_range_is_refused_in_a_view() { use asap_types::ir::operator::operator_properties::TimeShift; - use asap_types::ir::operator::Source; + use asap_types::ir::operator::{Reduction, Source}; use asap_types::ir::physical_export::compile_physical_asap_dag; - use asap_types::ir::schema::{DataType, Field, Schema}; + use asap_types::ir::scalar::ColumnRef; + use asap_types::ir::schema::{ + DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SummaryUpdate, + }; use asap_types::ir::{ apply_materialization_timings, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, }; + use asap_types::physical::Medium; use std::time::Duration; let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { @@ -914,7 +919,21 @@ mod tests { let install = |dag| { InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) }; - match install(export(shifted)) { + let summed = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: shifted, + family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::Named("value".into())), + reduction: Reduction::by(vec![]), + grouping: Default::default(), + filter: None, + })) + .unwrap(); + let mut assignment = MaterializationAssignment::default(); + assignment.set(&summed, Materialization::MatAtIngestion(Medium::Memory)); + let timed = + apply_materialization_timings(&summed, &assignment, &mut NodeToTimedNodeCache::new()) + .unwrap(); + match install(compile_physical_asap_dag(&timed).unwrap()) { Err(InstallablePlanError::Uninstallable( InstallableDagError::UnsupportedRawDataWindow { reason, .. }, )) => { diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index bb91f08fe..607259a63 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -12,3 +12,4 @@ mod runtime_install; mod sql_per_group_topk; mod sql_table_scans; mod summary_sharing; +mod time_operators; diff --git a/crates/planner/tests/time_operators.rs b/crates/planner/tests/time_operators.rs new file mode 100644 index 000000000..25b3df510 --- /dev/null +++ b/crates/planner/tests/time_operators.rs @@ -0,0 +1,412 @@ +//! Time operators planned exactly with `plan_installable` and run with the +//! Planner's executor over raw data: PromQL subqueries, `@` anchors (with +//! and without `offset`, `start()`/`end()` from the query range) and a +//! SQL `asap_rate`, whose range reads projected rows rather than a `Scan`. +//! Each answer is checked against the value the PromQL definition gives +//! for the same data: a range `(t - r, t]` is left-open, a subquery +//! `[R:s]` evaluates at the multiples of `s` in `(t - R, t]`, and `@` +//! pins the evaluation time (`start()` and `end()` the query range's +//! bounds). + +use std::collections::BTreeMap; +use std::sync::Arc; + +use asap_executor::physical_planner::compile; +use asap_executor::runtime::{Limits, RunContext, Scope}; +use asap_executor::sources::InMemoryDataStore; +use asap_executor::values::{Batch, Value}; +use asap_planner::{ + plan_installable, test_support::query_time_reference, FrontendInput, InstallablePlan, + PlanningModels, PlanningWorkload, SqlCatalog, UserInput, +}; +use asap_types::ir::operator::Source; +use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::NonASAPOp; +use futures::{executor::block_on, StreamExt}; + +use crate::time_series::{series_labels, time_series_tables, RawDataSample}; + +/// The evaluation time of most runs, a multiple of every step used. +const T: i64 = 1_800_000_000_000; +const SECOND: i64 = 1_000; + +/// No approximation. +const EXACT: &str = r#"{"max_error": 0.0, "failure_probability": 0.0}"#; + +/// One repeating `query` in `language` (its workload JSON), every 10 s, +/// exact, over `source`: 10 series sampled every 10 s, 1,000 rows held. +fn workload(language: &str, query: &str, source: &str) -> PlanningWorkload { + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": {language}, "repeating_queries": [ + {{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, + "requirements": {{"accuracy": {EXACT}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100.0}}}}, + "sources": {{"{source}": {{"series_count": 10, "sample_interval": 10000, + "rows_held": 1000}}}}}}}}"# + )) + .unwrap() +} + +/// The batches of `plan`'s one query, run at query time at +/// `evaluation_ms`, within the query range `range` (an instant query +/// without one), over `store`. +fn run( + plan: &InstallablePlan, + evaluation_ms: i64, + range: Option<(i64, i64)>, + store: impl FnOnce(&asap_planner::PhysicalASAPDAG) -> InMemoryDataStore, +) -> Vec { + let reference = query_time_reference(plan).unwrap(); + let dag = &reference.dag; + let roots = [dag.roots[0] as u64]; + let compiled = compile(dag, &[], &roots).expect("the executor compiles the plan"); + let instance = compiled.instantiate(Arc::new(store(dag))).unwrap(); + let mut context = RunContext::new( + Scope::Query { + evaluation_time_ms: evaluation_ms, + }, + Limits::default(), + ) + .unwrap(); + if let Some((start, end)) = range { + context = context.with_query_range(start, end).unwrap(); + } + block_on(async { + let mut output = instance + .execute(compiled.roots(), context) + .unwrap() + .remove(0); + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + batches.push((*batch.expect("the query runs")).clone()); + } + batches + }) +} + +// ── PromQL ─────────────────────────────────────────────────────────────── + +/// One sample of `metric` of job `job` at `timestamp_ms`. +fn sample(metric: &str, job: &str, timestamp_ms: i64, value: f64) -> RawDataSample { + RawDataSample { + labels: BTreeMap::from([ + ("__name__".to_string(), metric.to_string()), + ("job".to_string(), job.to_string()), + ]), + timestamp_ms, + value, + } +} + +async fn plan_promql(query: &str, metric: &str) -> InstallablePlan { + plan_installable(UserInput::new( + &workload(r#""promql""#, query, metric), + FrontendInput::Promql, + PlanningModels::builtin(), + )) + .await + .unwrap_or_else(|error| panic!("{query}: {error}")) + .0 +} + +/// Each answer of the PromQL `query` at `evaluation_ms` (within `range`) +/// over `samples`: its job and its value. +async fn promql_at( + query: &str, + metric: &str, + samples: &[RawDataSample], + evaluation_ms: i64, + range: Option<(i64, i64)>, +) -> Vec<(String, f64)> { + let plan = plan_promql(query, metric).await; + let mut answers: Vec<_> = run(&plan, evaluation_ms, range, |dag| { + time_series_tables(dag, samples) + }) + .iter() + .flat_map(|batch| { + batch.to_rows().into_iter().map(|row| { + let value = row + .iter() + .rev() + .find_map(|v| match v { + Value::Float64(v) => Some(*v), + _ => None, + }) + .expect("a sample value"); + let job = series_labels(batch.schema(), &row)["job"].clone(); + (job, value) + }) + }) + .collect(); + answers.sort_by(|a, b| a.0.cmp(&b.0)); + answers +} + +async fn promql(query: &str, metric: &str, samples: &[RawDataSample]) -> Vec<(String, f64)> { + promql_at(query, metric, samples, T, None).await +} + +fn assert_close(query: &str, got: &[(String, f64)], expected: &[(&str, f64)]) { + assert_eq!( + got.len(), + expected.len(), + "{query}: {got:?} vs {expected:?}" + ); + for ((job, got), (expected_job, value)) in got.iter().zip(expected) { + assert_eq!(job, expected_job, "{query}: {got:?} vs {expected:?}"); + assert!( + (got - value).abs() < 1e-9, + "{query}: job {job}: {got} vs {value}" + ); + } +} + +/// The gauge `g` of job `a`: 100 at `T - 600 s`, 1 at `T - 540 s`, 5 at +/// `T - 300 s`, 9 at `T - 30 s`, 50 at `T + 30 s`. The plan's instant +/// selector looks back 10 s (the sources' sample interval). +fn gauge() -> Vec { + [ + (-600, 100.0), + (-540, 1.0), + (-300, 5.0), + (-30, 9.0), + (30, 50.0), + ] + .into_iter() + .map(|(seconds, value)| sample("g", "a", T + seconds * SECOND, value)) + .collect() +} + +/// `g[10m:1m]` at `T + 30 s` evaluates `g` at the multiples of 1 minute in +/// `(T - 570 s, T + 30 s]`: `T - 540 s` … `T`. No step sees the 100 at +/// `T - 600 s` (before the grid), the 9 at `T - 30 s` (more than 10 s +/// before a step) or the 50 at `T + 30 s` (after the last step): two +/// steps see a sample, the maximum is the 5 at `T - 300 s` and the minimum +/// the 1 at `T - 540 s`. +#[tokio::test] +async fn subquery_of_a_selector_evaluates_at_aligned_steps() { + for (query, expected) in [ + ("max_over_time(g[10m:1m])", 5.0), + ("min_over_time(g[10m:1m])", 1.0), + ("count_over_time(g[10m:1m])", 2.0), + ] { + let got = promql_at(query, "g", &gauge(), T + 30 * SECOND, None).await; + assert_close(query, &got, &[("a", expected)]); + } +} + +/// The counter `c` of job `a`, sampled every 10 s from `T - 700 s` to `T`, +/// starting at 1000: it grows 1 per sample, 6 per sample in `(T - 300 s, +/// T - 240 s]` and 100 per sample in `(T - 660 s, T - 600 s]`. +fn counter() -> Vec { + let mut value = 1000.0; + (0..=70) + .map(|k| { + let seconds = -700 + 10 * k; + value += match seconds { + -295..=-240 => 6.0, + -655..=-600 => 100.0, + _ => 1.0, + }; + sample("c", "a", T + seconds * SECOND, value) + }) + .collect() +} + +/// `rate(c[1m])[10m:1m]` at `T` evaluates the rate at `T - 540 s` … `T`. +/// Each window `(t - 60 s, t]` holds 6 samples 10 s apart, so the rate +/// extrapolates their increase by 60/50: 1 per sample gives 0.1/s, the 6 +/// per sample at the step `T - 240 s` 0.6/s. The step `T - 600 s` (where +/// the rate is 10/s) is not on the left-open grid. With `offset 5m` the +/// grid is `T - 840 s` … `T - 300 s`: it holds `T - 600 s`. +#[tokio::test] +async fn subquery_of_a_rate_evaluates_at_steps_and_moves_with_offset() { + let samples = counter(); + for (query, expected) in [ + ("max_over_time(rate(c[1m])[10m:1m])", 0.6), + ("min_over_time(rate(c[1m])[10m:1m])", 0.1), + ("max_over_time(rate(c[1m])[10m:1m] offset 5m)", 10.0), + ] { + let got = promql(query, "c", &samples).await; + assert_close(query, &got, &[("a", expected)]); + } +} + +/// The gauge `h` of job `a` around `A = T - 1 h`: 3 at `A - 70 s`, 6 at +/// `A - 60 s`, 2 at `A - 5 s`, 4 at `A`, 99 at `A + 5 s`; and 7 at `T`. +fn anchored_gauge() -> Vec { + let anchor = T - 3_600 * SECOND; + let mut samples: Vec<_> = [(-70, 3.0), (-60, 6.0), (-5, 2.0), (0, 4.0), (5, 99.0)] + .into_iter() + .map(|(seconds, value)| sample("h", "a", anchor + seconds * SECOND, value)) + .collect(); + samples.push(sample("h", "a", T, 7.0)); + samples +} + +/// `h @ A` at `T` is `h` evaluated at `A`: the latest sample in `(A - 10 s, +/// A]`, the 4 at `A` (a sample at the window's end is in it, the 99 after +/// it is not). `offset 1m` moves the window to `(A - 70 s, A - 60 s]`: +/// the 6 at `A - 60 s`, not the 3 on its open start. Over a range the +/// anchored window reads `(A - 5 m, A]`. +#[tokio::test] +async fn an_at_timestamp_pins_the_selector_window() { + let anchor_s = (T - 3_600 * SECOND) / SECOND; + let samples = anchored_gauge(); + for (query, expected) in [ + (format!("h @ {anchor_s}"), 4.0), + (format!("h offset 1m @ {anchor_s}"), 6.0), + (format!("max_over_time(h[5m] @ {anchor_s})"), 6.0), + (format!("min_over_time(h[5m] @ {anchor_s})"), 2.0), + (format!("count_over_time(h[1m] @ {anchor_s})"), 2.0), + ] { + let got = promql(&query, "h", &samples).await; + assert_close(&query, &got, &[("a", expected)]); + } +} + +/// A counter's window ends at its `@` anchor: `rate(c[5m] @ t)` at `T` is +/// `rate(c[5m])` at `t`; `@ end()` and `@ start()` are the query range's +/// bounds, at every step of it (each step evaluates the same window), and +/// an instant query's evaluation time without a range. +#[tokio::test] +async fn an_anchored_counter_window_ends_at_the_anchor() { + let samples = counter(); + let rate_at = |at| async move { promql_at("rate(c[5m])", "c", &counter(), at, None).await }; + let at_240 = rate_at(T - 240 * SECOND).await; + let at_100 = rate_at(T - 100 * SECOND).await; + let at_t = rate_at(T).await; + let t_240_s = (T - 240 * SECOND) / SECOND; + let anchored = promql(&format!("rate(c[5m] @ {t_240_s})"), "c", &samples).await; + assert_eq!(anchored, at_240); + let anchored = promql( + &format!("rate(c[5m] @ {t_240_s} offset 140s)"), + "c", + &samples, + ) + .await; + assert_eq!(anchored, rate_at(T - 380 * SECOND).await); + let range = Some((T - 100 * SECOND, T)); + for step in [T - 100 * SECOND, T - 50 * SECOND, T] { + let end = promql_at("rate(c[5m] @ end())", "c", &samples, step, range).await; + assert_eq!(end, at_t, "@ end() at step {step}"); + let start = promql_at("rate(c[5m] @ start())", "c", &samples, step, range).await; + assert_eq!(start, at_100, "@ start() at step {step}"); + } + let instant = promql("rate(c[5m] @ start())", "c", &samples).await; + assert_eq!(instant, at_t); +} + +// ── SQL ────────────────────────────────────────────────────────────────── + +/// `metrics(ts, service, latency)`, one series per service. +fn metrics_schema() -> Schema { + Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![vec![0, 1]], + ) +} + +/// Each answer row of the SQL `query` at `T` over `rows` of `metrics` +/// `(seconds before T, service, latency)`. +async fn sql(query: &str, rows: &[(i64, &str, f64)]) -> Vec> { + let catalog = SqlCatalog::new().with_table("metrics", metrics_schema()); + let plan = plan_installable(UserInput::new( + &workload(r#"{"sql": "datafusion_sql"}"#, query, "metrics"), + FrontendInput::Sql { catalog: &catalog }, + PlanningModels::builtin(), + )) + .await + .unwrap_or_else(|error| panic!("{query}: {error}")) + .0; + let mut answers: Vec<_> = run(&plan, T, None, |dag| { + let schema = dag + .nodes + .iter() + .find_map(|node| match &node.payload { + PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { + Some(Arc::new(schema.clone())) + } + _ => None, + }) + .expect("the plan scans the table"); + let rows = rows + .iter() + .map(|&(seconds, service, latency)| { + schema + .fields + .iter() + .map(|field| match field.name.as_str() { + "ts" => Value::Timestamp(T - seconds * SECOND), + "service" => Value::Utf8(service.into()), + "latency" => Value::Float64(latency), + other => panic!("column {other}"), + }) + .collect() + }) + .collect(); + let batch = Batch::try_new(schema.clone(), rows).unwrap(); + InMemoryDataStore::default() + .with_raw_data( + Source::Table { + table_ref: "metrics".into(), + }, + schema, + vec![batch], + ) + .unwrap() + }) + .iter() + .flat_map(Batch::to_rows) + .collect(); + answers.sort_by(|a, b| format!("{a:?}").cmp(&format!("{b:?}"))); + answers +} + +/// `asap_rate(latency, ts, 300000)` is `rate(latency[5m])` per service: a +/// counter from 100 sampled every 10 s in `(T - 300 s, T]` grows 58 in +/// 290 s (2 per sample for `a`, 3 for `b`), extrapolated by 300/290 to +/// the window's start, so 0.2/s and 0.3/s. The 0 at `T - 300 s` lies on +/// the window's open start: read, it would be a counter reset. +#[tokio::test] +async fn sql_asap_rate_reads_its_window_of_projected_rows() { + let mut rows = vec![(300, "a", 0.0), (300, "b", 0.0)]; + for k in 0..30 { + rows.push((290 - 10 * k, "a", 100.0 + 2.0 * k as f64)); + rows.push((290 - 10 * k, "b", 100.0 + 3.0 * k as f64)); + } + let answers = sql( + "SELECT service, asap_rate(latency, ts, 300000) AS r FROM metrics GROUP BY service", + &rows, + ) + .await; + let rates: Vec<(String, f64)> = answers + .iter() + .map(|row| { + let service = row + .iter() + .find_map(|v| match v { + Value::Utf8(s) => Some(s.to_string()), + _ => None, + }) + .expect("a service"); + let rate = row + .iter() + .rev() + .find_map(|v| match v { + Value::Float64(v) => Some(*v), + _ => None, + }) + .expect("a rate"); + (service, rate) + }) + .collect(); + assert_close("asap_rate", &rates, &[("a", 0.2), ("b", 0.3)]); +} diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index 6a3c2de29..758ec41a8 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -332,8 +332,8 @@ pub enum InstallableDagError { /// Bad input: the DAG is not a valid physical ASAP DAG. #[error("invalid physical ASAP DAG: {0}")] InvalidDag(#[from] PhysicalASAPDAGValidationError), - /// Unsatisfiable request: the DAG reads raw data through `TimeRange` - /// `time_range_node_id` in a shape whose window is not a fixed offset + /// Unsatisfiable request: a materialized view's refresh reads raw data + /// through `TimeRange` `time_range_node_id` in a shape whose window is not a fixed offset /// from the evaluation time (`reason`: a subquery or a `TimeShift` /// above it, or an `@` anchor below it), or that ends after the /// evaluation (a negative offset below it), which no ingestion run can @@ -701,19 +701,45 @@ impl PhysicalASAPDAG { } /// [`Self::scan_window`] of each query-time `Scan`, with no chain kept, - /// by id. + /// by id; `Unbounded` for a `Scan` read through a `TimeRange` that a + /// subquery or a `TimeShift` above it evaluates at other times than the + /// evaluation time (`max_over_time(rate(m[1m])[10m:1m])` reads `m` 11 + /// minutes back, not 1). fn query_time_scan_windows(&self) -> HashMap { let scans = self .nodes .iter() .filter(|node| is_query_time_scan(node)) .map(|node| node.id); - self.scan_windows(scans, &HashSet::new()) + let mut windows = self.scan_windows(scans, &HashSet::new()); + let index = ViewIndex::new(self); + for node in &self.nodes { + if !matches!(node.payload, Operator::NonASAP(NonASAPOp::TimeRange { .. })) + || !index.re_timed_from_above.contains(&node.id) + { + continue; + } + let mut below = index.only_input(node.id); + while let Some(id) = below { + match index.relational(id) { + Some(NonASAPOp::TimeShift { .. }) => below = index.only_input(id), + Some(NonASAPOp::Scan { .. }) => { + if let Some(window) = windows.get_mut(&id) { + *window = ScanWindow::Unbounded; + } + break; + } + _ => break, + } + } + } + windows } - /// Checks that every raw-data window the DAG reads (each `TimeRange`, - /// and each `SummaryTumblingWindowChain`'s tumbling window, reading a `Scan` without passing another - /// `TimeRange`) has a supported shape + /// Checks that every raw-data window a materialized view's refresh + /// reads (each `TimeRange`, and each `SummaryTumblingWindowChain`'s + /// tumbling window, reading a `Scan` without passing another + /// `TimeRange`, below a view) has a supported shape /// ([`InstallableDagError::UnsupportedRawDataWindow`]). O(nodes + edges) /// plus the depth below each window. pub fn check_raw_data_windows(&self) -> Result<(), InstallableDagError> { @@ -854,6 +880,8 @@ struct ViewIndex<'a> { /// The nodes with a `TimeShift`, `TimeRange` or `PromqlSubquery` above /// them, which evaluates them at other times than the evaluation time. re_timed_from_above: HashSet, + /// The nodes below a materialized view, which its refresh computes. + refreshed_by_a_view: HashSet, } /// Whether `operator` evaluates its input at other times than its own. @@ -911,9 +939,18 @@ impl<'a> ViewIndex<'a> { } } let mut re_timed_from_above = HashSet::new(); + let mut refreshed_by_a_view = HashSet::new(); for id in children_first.iter().rev() { + let below = inputs.get(id).into_iter().flatten().copied(); if re_timed_from_above.contains(id) || payload(id).is_some_and(re_times_its_input) { - re_timed_from_above.extend(inputs.get(id).into_iter().flatten().copied()); + re_timed_from_above.extend(below.clone()); + } + if refreshed_by_a_view.contains(id) + || nodes + .get(id) + .is_some_and(|node| node.materialization.is_materialized_view()) + { + refreshed_by_a_view.extend(below); } } ViewIndex { @@ -923,6 +960,7 @@ impl<'a> ViewIndex<'a> { children_first, reading_raw_data, re_timed_from_above, + refreshed_by_a_view, } } @@ -943,7 +981,11 @@ impl<'a> ViewIndex<'a> { /// See [`PhysicalASAPDAG::check_raw_data_windows`]. fn check_raw_data_windows(&self) -> Result<(), InstallableDagError> { for node in &self.dag.nodes { - if !self.reading_raw_data.contains(&node.id) { + // A query run reads any window shape: the executor reads the + // `Scan` through its `scan_window`. + if !self.reading_raw_data.contains(&node.id) + || !self.refreshed_by_a_view.contains(&node.id) + { continue; } match &node.payload {