Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions crates/executor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ consumer), as its `Lifetime` and the run's scope say:
one consumer: the node runs once, to the end, before any consumer reads
it, and every consumer reads its finished output independently, as the
planner charges one run whose output is held. Likewise a computed
`AcrossEvaluations` node with more than one consumer, which only a
hand-built DAG has at query time (a compiled plan reads it through its
view slot). For example, a `OneEvaluation` scan read by three queries
is one store read. The output is charged to the run's byte budget until
`AcrossEvaluations` node with more than one consumer: an unbound view
(a compile with its view slot not bound computes it for its readers in
the query run), or a node of a hand-built DAG. For example, a
`OneEvaluation` scan read by three queries is one store read. The output is charged to the run's byte budget until
every consumer has read it; over the budget the run fails with
`MemoryLimit` (nothing spills). With one consumer the node streams to it,
since no sibling can wait on that consumer.
Expand Down
1 change: 1 addition & 0 deletions crates/executor/src/operators/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ pub use aggregate::Reduction;
pub use series_window::SubquerySteps;
pub use sort::SortKey;
pub use summary::{RowWeight, SummaryEvaluation};
pub(crate) use time_range::TUMBLING_WINDOW_END_COLUMN;
#[derive(Clone, serde::Serialize, serde::Deserialize)]
enum Kind {
#[serde(skip)]
Expand Down
166 changes: 143 additions & 23 deletions crates/executor/src/operators/time_range.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,15 @@ impl Operator {
})
}

/// This range window (not an instant window), each kept row's time
/// replaced by the end of the tumbling window of `width_ms` it lies in,
/// the windows counted back from the range's end `E`: `(E - (k+1) ·
/// width_ms, E - k · width_ms]` ends at `E - k · width_ms`. A
/// tumbling-window chain built at query time groups its rows by that
/// end. Example: with `E` = 600,000 and windows of 60,000 ms, rows at
/// 540,001 and 600,000 end at 600,000, a row at 540,000 at 540,000.
/// This range window (not an instant window), each kept row followed
/// by the end of the tumbling window of `width_ms` its time lies in, in
/// a trailing [`TUMBLING_WINDOW_END_COLUMN`]; the row's own time is
/// kept, as a counter's samples need it. The windows count back from
/// the range's end `E`: `(E - (k+1) · width_ms, E - k · width_ms]` ends
/// at `E - k · width_ms`. A tumbling-window chain built for its readers
/// groups its rows by that end. Example: with `E` = 600,000 and windows
/// of 60,000 ms, rows at 540,001 and 600,000 end at 600,000, a row at
/// 540,000 at 540,000.
pub fn with_tumbling_window_ends(mut self, width_ms: i64) -> Result<Self, Error> {
let Kind::TimeRange {
latest_per_series: None,
Expand All @@ -73,11 +75,30 @@ impl Operator {
if width_ms <= 0 {
return Err(invalid("a tumbling window has a positive width"));
}
if self
.output
.fields
.iter()
.any(|field| field.name == TUMBLING_WINDOW_END_COLUMN)
{
return Err(invalid("a range window's input already has a window end"));
}
*slot = Some(width_ms);
let mut output = (*self.output).clone();
output.fields.push(result_field(
TUMBLING_WINDOW_END_COLUMN,
DataType::Timestamp,
false,
));
self.output = Arc::new(output);
Ok(self)
}
}

/// The column [`Operator::with_tumbling_window_ends`] appends: each row's
/// tumbling window end.
pub(crate) const TUMBLING_WINDOW_END_COLUMN: &str = "$tumbling_window_end";

impl Operator {
/// A `TimeShift { offset_ms, at }`: its rows are its input's, which a
/// run evaluates once, at the anchor's time minus `offset_ms`
Expand Down Expand Up @@ -228,7 +249,7 @@ pub(super) fn execute<'a>(
batch.filter(&keep)?
};
match tumbling_window_width_ms {
Some(width_ms) => window_ends(batch, time, end, width_ms),
Some(width_ms) => window_ends(batch, &operator.output, time, end, width_ms),
None => Ok(batch),
}
})
Expand Down Expand Up @@ -274,10 +295,16 @@ pub(super) fn execute<'a>(
.boxed_local())
}

/// `batch` (its rows in `(.., end]`) with each row's `time` replaced by the
/// end of its tumbling window of `width_ms` counted back from `end`
/// ([`Operator::with_tumbling_window_ends`]).
fn window_ends(batch: Batch, time: usize, end: i64, width_ms: i64) -> Result<Batch, Error> {
/// `batch` (its rows in `(.., end]`) as `output`: each row followed by the
/// end of its `time`'s tumbling window of `width_ms` counted back from
/// `end` ([`Operator::with_tumbling_window_ends`]).
fn window_ends(
batch: Batch,
output: &SchemaRef,
time: usize,
end: i64,
width_ms: i64,
) -> Result<Batch, Error> {
let times = batch
.column(time)
.as_any()
Expand All @@ -290,16 +317,109 @@ fn window_ends(batch: Batch, time: usize, end: i64, width_ms: i64) -> Result<Bat
.map(|t| t.map(|t| end - (end - t) / width_ms * width_ms))
.collect();
let rows = batch.num_rows();
let mut arrays = Vec::with_capacity(batch.schema().fields.len());
let mut states = Vec::with_capacity(arrays.capacity());
for column in 0..batch.schema().fields.len() {
if column == time {
arrays.push(std::sync::Arc::new(ends.clone()) as arrow::array::ArrayRef);
states.push(None);
} else {
arrays.push(batch.column(column).clone());
states.push(batch.state_column(column).cloned());
}
let mut arrays = batch.record_batch().columns().to_vec();
let mut states: Vec<_> = (0..arrays.len())
.map(|column| batch.state_column(column).cloned())
.collect();
arrays.push(std::sync::Arc::new(ends) as arrow::array::ArrayRef);
states.push(None);
Batch::from_parts(output.clone(), arrays, states, rows)
}

#[cfg(test)]
mod tests {
use super::*;
use crate::plan::PhysicalDAG;
use crate::runtime::{Limits, Scope};
use futures::executor::block_on;
use planner_types::stages::Lifetime;

const MINUTE_MS: i64 = 60_000;

// A range window with tumbling-window ends: at T = 10m, `(T - 3m, T]`
// in windows of 1 minute keeps the rows at 7m (out), 7m + 1 ms, 8m,
// 9m 30 s and 10m, each keeping its time and followed, in
// `TUMBLING_WINDOW_END_COLUMN`, by its window's end: 8m, 8m, 10m and
// 10m.
#[test]
fn a_range_window_stamps_each_row_with_its_tumbling_windows_end() {
let schema = Arc::new(Schema::with_time_index(
vec![
SummaryField::plain("job", DataType::Utf8, false),
SummaryField::plain("ts", DataType::Timestamp, false),
SummaryField::plain("value", DataType::Float64, false),
],
1,
vec![],
));
let times = [
7 * MINUTE_MS,
7 * MINUTE_MS + 1,
8 * MINUTE_MS,
570_000,
10 * MINUTE_MS,
];
let rows = times
.iter()
.map(|&t| {
vec![
Value::Utf8("a".into()),
Value::Timestamp(t),
Value::Float64(1.0),
]
})
.collect();
let batch = Batch::try_new(schema.clone(), rows).unwrap();
let mut dag = PhysicalDAG::default();
dag.add(
0,
vec![],
Operator::source(schema.clone(), vec![batch]).unwrap(),
Lifetime::None,
)
.unwrap();
let window = Operator::time_range(schema, 1, 0, 3 * MINUTE_MS, None, None)
.unwrap()
.with_tumbling_window_ends(MINUTE_MS)
.unwrap();
let output = window.schema();
let end = output
.fields
.iter()
.position(|field| field.name == TUMBLING_WINDOW_END_COLUMN)
.unwrap();
dag.add(1, vec![0], window, Lifetime::None).unwrap();
let context = RunContext::new(
Scope::Query {
evaluation_time_ms: 10 * MINUTE_MS,
},
Limits::default(),
)
.unwrap();
let mut outputs = dag.execute(&[1], context).unwrap();
let rows = block_on(async {
let mut rows = Vec::new();
while let Some(batch) = outputs[0].next().await {
rows.extend(batch.unwrap().to_rows());
}
rows
});
// Each kept row's own time and, appended, its window's end.
let times_and_ends: Vec<(i64, i64)> = rows
.into_iter()
.map(|row| match (&row[1], &row[end]) {
(Value::Timestamp(time), Value::Timestamp(end)) => (*time, *end),
other => panic!("{other:?}"),
})
.collect();
assert_eq!(
times_and_ends,
[
(7 * MINUTE_MS + 1, 8 * MINUTE_MS),
(8 * MINUTE_MS, 8 * MINUTE_MS),
(570_000, 10 * MINUTE_MS),
(10 * MINUTE_MS, 10 * MINUTE_MS),
]
);
}
Batch::from_parts(batch.schema().clone(), arrays, states, rows)
}
Loading
Loading