Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions crates/cost/src/usage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -424,8 +424,8 @@ fn each_node_usage<'a>(
let scan_window = match (&node.payload, node.output_state.timing) {
(Payload::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => {
match dag.scan_window(node.id, kept_chains) {
// Priced as before #955 decides `@` anchors and
// unwindowed readers: the widest window read.
// An `@` anchor or an unwindowed reader: priced at
// the widest window read.
ScanWindow::Unbounded => trailing_window(
widest_window_read_ms(dag, nodes, node.id, kept_chains)
.or(root_scan_lookback_ms),
Expand Down Expand Up @@ -1617,11 +1617,12 @@ pub fn kept_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet<PhysicalASA
}

/// The widest window, in milliseconds, read above the scan `scan` at query
/// time, ignoring readers that read it outside a window and the anchors of
/// shifts: each range `TimeRange` and `SummaryTumblingWindowChain` (its whole window, or
/// one tumbling window for a chain in `kept_chains`) reached through time shifts.
/// Prices a scan whose [`PhysicalASAPDAG::scan_window`] is unbounded as
/// Stage 3 did before; #955 decides `@` anchors.
/// time, ignoring readers that read it outside a window: each range
/// `TimeRange` and `SummaryTumblingWindowChain` (its whole window, or one
/// tumbling window for a chain in `kept_chains`) reached through time
/// shifts without an `@` anchor. A read through an `@` anchor is outside a
/// window, as Stage 3's selection MILP prices it. Prices a scan whose
/// [`PhysicalASAPDAG::scan_window`] is unbounded as Stage 3 did before.
fn widest_window_read_ms(
dag: &PhysicalASAPDAG,
nodes: &NodeMap<'_>,
Expand All @@ -1647,7 +1648,7 @@ fn widest_window_read_ms(
tumbling_window_count,
..
}) => tumbling_window_width_ms.saturating_mul(u64::from(*tumbling_window_count)),
Payload::NonASAP(NonASAPOp::TimeShift { .. }) => {
Payload::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => {
stack.push(edge.consumer);
continue;
}
Expand Down
24 changes: 6 additions & 18 deletions crates/devtools/tests/memo_pricing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -339,12 +339,12 @@ fn scalar_root_choices_are_composed_and_priced() {
assert!(priced.iter().any(|p| p.valid));
}

/// A choice whose selected plan installation refuses for a reason other
/// than the executor (here a raw-data window under a subquery, which no
/// runtime can place) is `not_installable`, never valid, so it is never
/// flagged cheaper than the selected plan.
/// A subquery's choices install: the executor reads a subquery's raw
/// samples at query time, so every choice of
/// `max_over_time(rate(data[5m])[1h:1m])` is valid, none cheaper than the
/// selected plan.
#[test]
fn a_choice_installation_refuses_is_not_installable() {
fn subquery_choices_install() {
let mut workload = ratio_of_quantiles();
workload.query_workload.repeating_queries.as_mut().unwrap()[0].query =
Query("max_over_time(rate(data[5m])[1h:1m])".into());
Expand All @@ -368,18 +368,6 @@ fn a_choice_installation_refuses_is_not_installable() {
);
assert!(!priced.is_empty());
for p in &priced {
assert!(!p.valid && !p.cheaper_than_selected, "{p:?}");
}
let refused: Vec<_> = priced
.iter()
.filter(|p| p.rejection.as_deref() == Some("not_installable"))
.collect();
assert!(!refused.is_empty(), "{priced:?}");
for p in refused {
let reason = p.rejection_reason.as_deref().unwrap();
assert!(
reason.contains("a subquery or a TimeShift above the range"),
"{reason}"
);
assert!(p.valid && !p.cheaper_than_selected, "{p:?}");
}
}
30 changes: 22 additions & 8 deletions crates/executor/src/operators/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,11 @@ enum Kind {
},
Project(Vec<Expression>),
Filter(Expression),
/// A `TimeRange` over a `Scan` ([`Operator::time_range`]).
/// A `TimeRange` over a time-indexed input ([`Operator::time_range`]).
TimeRange {
time: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
at: Option<planner_types::ir::operator::AtModifier>,
end_offset_ms: i64,
width_ms: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
Expand Down Expand Up @@ -238,8 +240,14 @@ impl Operator {
}

/// Set a counter evaluation's logical window: `lookback` long, ending
/// `offset` before the evaluation time.
pub(crate) fn with_counter_window(mut self, offset: i64, lookback: i64) -> Result<Self, Error> {
/// `offset` before the evaluation time, or before the time of the `@`
/// anchor `at`.
pub(crate) fn with_counter_window(
mut self,
at: Option<planner_types::ir::operator::AtModifier>,
offset: i64,
lookback: i64,
) -> Result<Self, Error> {
if lookback <= 0 {
return Err(invalid("counter lookback must be positive"));
}
Expand All @@ -250,6 +258,7 @@ impl Operator {
{
evaluation.lookback_ms = Some(lookback);
evaluation.offset_ms = offset;
evaluation.at = at;
}
Ok(self)
}
Expand All @@ -264,18 +273,23 @@ impl Operator {
SummaryEvaluation::Exact(crate::summary_kernels::exact::ExactEvaluation {
lookback_ms: Some(lookback),
offset_ms,
at,
..
}),
..
} = &self.kind
else {
return Ok(None);
};
let end = match context.scope {
crate::runtime::Scope::Query {
evaluation_time_ms, ..
} => evaluation_time_ms,
crate::runtime::Scope::Ingestion { window_end_ms, .. } => window_end_ms,
let end = match (at, &context.scope) {
(Some(at), _) => time_range::anchor_time(*at, context)?,
(
None,
crate::runtime::Scope::Query {
evaluation_time_ms, ..
},
) => *evaluation_time_ms,
(None, crate::runtime::Scope::Ingestion { window_end_ms, .. }) => *window_end_ms,
}
.checked_sub(*offset_ms)
.ok_or_else(|| invalid("counter window overflows Int64"))?;
Expand Down
12 changes: 2 additions & 10 deletions crates/executor/src/operators/series_window.rs
Original file line number Diff line number Diff line change
Expand Up @@ -138,16 +138,8 @@ pub(super) fn anchor(
) -> Result<Option<i64>, Error> {
match bound {
None => Ok(literal),
Some(bound) => {
let (start, end) = context
.query_range()
.ok_or_else(|| invalid("@ start() and @ end() require query range bounds"))?;
match bound {
AtModifier::Start => Ok(Some(start)),
AtModifier::End => Ok(Some(end)),
AtModifier::Timestamp(_) => Err(invalid("invalid range-bound anchor")),
}
}
Some(AtModifier::Timestamp(_)) => Err(invalid("invalid range-bound anchor")),
Some(bound) => super::time_range::anchor_time(bound, context).map(Some),
}
}

Expand Down
87 changes: 62 additions & 25 deletions crates/executor/src/operators/time_range.rs
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
//! A `TimeRange` over a `Scan`: the rows of one window on the time column,
//! for a SQL table and a time series alike. A time series' stale markers
//! (the Prometheus end-of-series NaN) are never rows; an ordinary NaN is.
//! A `TimeRange` over a time-indexed input (a `Scan`, or rows computed
//! from one): the rows of one window on the time column, for a SQL table
//! and a time series alike. A time series' stale markers (the Prometheus
//! end-of-series NaN) are never rows; an ordinary NaN is.
use super::*;
use arrow::array::Array;
use planner_types::ir::operator::AtModifier;

impl Operator {
/// The rows of `input` whose `time` lies in the window of a
/// `TimeRange(width_ms)` over shifts summing to `end_offset_ms`: `(T -
/// end_offset_ms - width_ms, T - end_offset_ms]` in a query run at `T`,
/// `(window_end_ms - width_ms, window_end_ms]` in an ingestion run.
/// With `sample_value`, a time series' value column, a stale marker is
/// not a row. With `latest_per_series` (an instant window), only each
/// series' latest row in the window is kept (of rows at one time, the
/// last), and none when that row is a stale marker; a series is the
/// values of those columns (its labels).
/// `TimeRange(width_ms)` over shifts: `(e - width_ms, e]`, with `e` the
/// window end [`window_end`] gives for the shifts' `@` anchor `at` and
/// offsets summing to `end_offset_ms` (`T - end_offset_ms` in a query
/// run at `T` without an anchor, the run's `window_end_ms` in an
/// ingestion run). With `sample_value`, a time series' value column, a
/// stale marker is not a row. With `latest_per_series` (an instant
/// window), only each series' latest row in the window is kept (of rows
/// at one time, the last), and none when that row is a stale marker; a
/// series is the values of those columns (its labels).
pub fn time_range(
input: SchemaRef,
time: usize,
at: Option<AtModifier>,
end_offset_ms: i64,
width_ms: i64,
sample_value: Option<usize>,
Expand All @@ -39,6 +43,7 @@ impl Operator {
Ok(Self {
kind: Kind::TimeRange {
time,
at,
end_offset_ms,
width_ms,
sample_value,
Expand All @@ -50,24 +55,55 @@ impl Operator {
}
}

/// `(start, end]` of a window `width_ms` wide in this run: ending
/// `end_offset_ms` before the evaluation time `T` in a query run; ending at
/// the run's `window_end_ms` in an ingestion run, which builds the window
/// the shifts select (the shifts say which window a query reads, the
/// ingestion run's window which one it builds).
/// The time an `@` anchor pins: `@ <t>` its instant, `@ start()` and
/// `@ end()` the query range's bounds, which an instant query (a run
/// without them) has both at its evaluation time, as in Prometheus. An
/// ingestion run has no anchor time: an anchored window is read at query
/// time.
pub(super) fn anchor_time(at: AtModifier, context: &RunContext) -> Result<i64, Error> {
let crate::runtime::Scope::Query { evaluation_time_ms } = context.scope else {
return Err(invalid("an @-anchored window is read only at query time"));
};
let (start, end) = context
.query_range()
.unwrap_or((evaluation_time_ms, evaluation_time_ms));
Ok(match at {
AtModifier::Timestamp(ms) => ms,
AtModifier::Start => start,
AtModifier::End => end,
})
}

/// The end of a window `end_offset_ms` before its reference time: the time
/// of the `@` anchor `at` ([`anchor_time`]) or, without one, the evaluation
/// time `T` in a query run; in an ingestion run, the run's `window_end_ms`,
/// which builds the window the shifts select (the shifts say which window
/// a query reads, the ingestion run's window which one it builds).
pub(super) fn window_end(
at: Option<AtModifier>,
end_offset_ms: i64,
context: &RunContext,
) -> Result<i64, Error> {
let reference = match (at, &context.scope) {
(Some(at), _) => anchor_time(at, context)?,
(None, crate::runtime::Scope::Query { evaluation_time_ms }) => *evaluation_time_ms,
(None, crate::runtime::Scope::Ingestion { window_end_ms, .. }) => {
return Ok(*window_end_ms)
}
};
reference
.checked_sub(end_offset_ms)
.ok_or_else(|| invalid("time range overflows Int64"))
}

/// `(start, end]` of a window `width_ms` wide ending at [`window_end`].
pub(super) fn window(
at: Option<AtModifier>,
end_offset_ms: i64,
width_ms: i64,
context: &RunContext,
) -> Result<(i64, i64), Error> {
let end = match context.scope {
crate::runtime::Scope::Query {
evaluation_time_ms, ..
} => evaluation_time_ms
.checked_sub(end_offset_ms)
.ok_or_else(|| invalid("time range overflows Int64"))?,
crate::runtime::Scope::Ingestion { window_end_ms, .. } => window_end_ms,
};
let end = window_end(at, end_offset_ms, context)?;
let start = end
.checked_sub(width_ms)
.ok_or_else(|| invalid("time range overflows Int64"))?;
Expand All @@ -81,6 +117,7 @@ pub(super) fn execute<'a>(
) -> Result<OutputStream<'a, Batch>, Error> {
let Kind::TimeRange {
time,
at,
end_offset_ms,
width_ms,
sample_value,
Expand All @@ -92,7 +129,7 @@ pub(super) fn execute<'a>(
let input = inputs
.pop()
.ok_or_else(|| invalid("time range input missing"))?;
let (start, end) = window(end_offset_ms, width_ms, &context)?;
let (start, end) = window(at, end_offset_ms, width_ms, &context)?;
// Whether each row is in the window and, for a range window, not a
// stale marker.
let kept =
Expand Down
2 changes: 2 additions & 0 deletions crates/executor/src/operators/unchecked.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,13 +110,15 @@ impl TryFrom<UncheckedOperator> for Operator {
Kind::Filter(expression) => Operator::filter(input(0)?, expression)?,
Kind::TimeRange {
time,
at,
end_offset_ms,
width_ms,
sample_value,
latest_per_series,
} => Operator::time_range(
input(0)?,
time,
at,
end_offset_ms,
width_ms,
sample_value,
Expand Down
Loading
Loading