From 93f223af5b2fcfbb11adaf1024a69777d24b0e92 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:42:01 +0000 Subject: [PATCH 1/7] fix(executor): run each node by its plan Lifetime, so roots drain in any order A shared producer kept one queue bounded by `max_buffered_batches`; a reader ahead of the others waited for them, so draining root streams one after another (y16), or a SeriesBinary collecting its left branch before its right over one stored chain (b17, g10), hung forever. The runtime now runs each node by the plan's `Lifetime` and the run's scope: - query time, a leaf (raw data or view slot): one store read per consumer; - query time, OneEvaluation or AcrossEvaluations: computed to the end once, then read by every consumer, charged to the byte budget; - query time, None: one consumer; more is an error naming the node; - ingestion time: a node with several consumers runs once and each batch reaches every consumer; `execute_ingestion` polls every root and returns one future with each root's batches. `PhysicalDAG::add` takes the node's `Lifetime`; compiling a PhysicalASAPDAG passes each node's own. `Limits::max_buffered_batches`, reader positions and the batch-count backpressure are removed. Closes #1290 --- crates/executor/README.md | 39 +- .../src/operators/aggregate/temporal.rs | 16 +- .../executor/src/operators/series_labels.rs | 4 +- .../executor/src/physical_planner/compiled.rs | 49 ++- crates/executor/src/physical_planner/mod.rs | 78 +++- .../src/physical_planner/promql_values.rs | 5 +- crates/executor/src/plan/mod.rs | 33 +- crates/executor/src/runtime/context.rs | 4 +- crates/executor/src/runtime/mod.rs | 316 +++++++++++------ crates/executor/src/runtime/tests.rs | 333 ++++++++++++++---- .../executor/tests/aggregate_memory_limit.rs | 12 +- crates/executor/tests/blocking_resources.rs | 19 +- .../tests/cheap_measures_memory_limit.rs | 8 +- crates/executor/tests/common/mod.rs | 29 ++ crates/executor/tests/current_series_heap.rs | 12 +- crates/executor/tests/current_timestamp.rs | 29 +- crates/executor/tests/exact_state_columns.rs | 12 +- .../tests/grouped_limit_memory_limit.rs | 12 +- crates/executor/tests/hydra_cms_execution.rs | 9 +- .../tests/keyed_summary_row_weight.rs | 28 +- crates/executor/tests/physical_dag.rs | 153 +++++--- crates/executor/tests/physical_semantics.rs | 104 ++++-- .../executor/tests/precompute_population.rs | 17 +- crates/executor/tests/promql_binary.rs | 8 +- crates/executor/tests/promql_values.rs | 10 +- crates/executor/tests/raw_scan.rs | 31 +- .../tests/series_window_memory_limit.rs | 13 +- crates/executor/tests/state_column_updater.rs | 4 +- .../stored_tumbling_window_memory_limit.rs | 39 +- .../tests/summary_build_memory_limit.rs | 4 +- crates/executor/tests/text_extremum_memory.rs | 8 +- crates/executor/tests/time_series_window.rs | 24 +- .../executor/tests/tumbling_window_chain.rs | 30 +- crates/executor/tests/univmon_execution.rs | 10 +- .../tests/kll_tumbling_window_execution.rs | 21 +- .../tests/physical_common/mod.rs | 34 +- crates/planner/tests/columnar_batches.rs | 8 +- crates/planner/tests/sql_table_scans.rs | 33 +- crates/types/src/plan/physical_asap_dag.rs | 34 +- 39 files changed, 1119 insertions(+), 513 deletions(-) diff --git a/crates/executor/README.md b/crates/executor/README.md index de199a07f..c76670a39 100644 --- a/crates/executor/README.md +++ b/crates/executor/README.md @@ -6,12 +6,32 @@ a storage implementation nor DataFusion. Batches are Arrow columns (`values::Batch`); DataFusion informed the design, it is not the execution framework. -`plan::PhysicalDAG` binds typed operator inputs to node IDs. Each execution starts -one producer per reachable node, shares output batches among its consumers, and -bounds buffering. Dropping one consumer does not cancel other consumers. A -`RunContext` carries query or ingestion scope, cancellation and byte accounting. -Executions use the caller's worker and worker-local streams, with no internal -thread pool. Poll multiple root streams concurrently when they share inputs. +`plan::PhysicalDAG` binds typed operator inputs to node IDs, each node with the +plan's `Lifetime`. A `RunContext` carries query or ingestion scope, +cancellation and byte accounting. Executions use the caller's worker and +worker-local streams, with no internal thread pool. + +A node runs once per evaluation time in a run (an `@` anchor evaluates the +nodes below it at another time), as its `Lifetime` and the run's scope say: + +- Query time, a leaf (a raw data or materialized view slot, a store read): + each consumer reads the store on its own, as the planner charges a + not-materialized read on each edge. +- Query time, `OneEvaluation` or `AcrossEvaluations`: the node runs to the + end before any consumer reads it, and every consumer reads its finished + output independently. The output is charged to the run's byte budget until + every consumer has read it; over the budget the run fails with + `MemoryLimit` (nothing spills). +- Query time, `None`: streamed to its one consumer. A computed node with more + than one consumer is a planner bug; `execute` returns an error naming the + node before any operator starts. +- Ingestion time, a node with several consumers (one per-row node feeding + several chain builds or views): the node runs once and each batch reaches + every consumer. `execute_ingestion` polls every root itself and returns one + future that resolves to each root's batches, for the caller to write. + +`execute` runs a query and returns one stream per root; the caller may drain +them in any order. Dropping one consumer does not cancel other consumers. `operators::Operator` implements native batch sources, scalar values, projection, filtering, grouped exact aggregation, semi-join, grouped Sort and @@ -28,6 +48,7 @@ use asap_executor::{ plan::PhysicalDAG, runtime::{Limits, RunContext, Scope}, }; +use asap_executor::planner::stages::Lifetime; use asap_executor::planner::ir::schema::DataType; use futures::{executor::block_on, StreamExt}; @@ -36,8 +57,8 @@ let negate = Operator::project(source.schema(), vec![ ("value".into(), Expression::Negate(Box::new(Expression::Column(0)))), ])?; let mut plan = PhysicalDAG::default(); -plan.add(0, vec![], source)?; -plan.add(1, vec![0], negate)?; +plan.add(0, vec![], source, Lifetime::None)?; +plan.add(1, vec![0], negate, Lifetime::None)?; let run = RunContext::new( Scope::Query { evaluation_time_ms: 1000 }, Limits::default(), @@ -76,7 +97,7 @@ See [the design](../../docs/design_docs/physical-planning-and-deployment.md). ## Module boundaries - `plan`: immutable DAG, operator interface, schemas and execution properties. -- `runtime`: per-run streams, shared producers, memory reservations and cancellation. +- `runtime`: per-run streams, how each node runs by its `Lifetime`, memory reservations and cancellation. - `expressions`: scalar evaluation; typed builders and the Planner expression adapter. - `operators`: projection, filter, joins, aggregate/window, sort, limit and summary implementations. - `sources`: the `DataStore` data interface, `Scan` and the in-memory store. diff --git a/crates/executor/src/operators/aggregate/temporal.rs b/crates/executor/src/operators/aggregate/temporal.rs index 12771677e..02df9ec8e 100644 --- a/crates/executor/src/operators/aggregate/temporal.rs +++ b/crates/executor/src/operators/aggregate/temporal.rs @@ -326,6 +326,7 @@ mod tests { values::Batch, }; use futures::StreamExt; + use planner_types::stages::Lifetime; use planner_types::ir::schema::{ DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, @@ -383,12 +384,21 @@ mod tests { 0, vec![], Operator::source(schema.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], operator).unwrap(); + dag.add(1, vec![0], operator, Lifetime::None).unwrap(); let context = RunContext::new(scope.clone(), Limits::default()).unwrap(); - let mut output = dag.execute(&[1], context).unwrap().remove(0); - let result = futures::executor::block_on(output.next()).unwrap().unwrap(); + let result = if matches!(scope, Scope::Ingestion { .. }) { + let run = dag.execute_ingestion(&[1], context).unwrap(); + futures::executor::block_on(run) + .unwrap() + .remove(0) + .remove(0) + } else { + let mut output = dag.execute(&[1], context).unwrap().remove(0); + futures::executor::block_on(output.next()).unwrap().unwrap() + }; assert_eq!( format!("{:?}", result.to_rows()[0][0]), format!("{expected:?}") diff --git a/crates/executor/src/operators/series_labels.rs b/crates/executor/src/operators/series_labels.rs index 2bce411a4..1fee5822e 100644 --- a/crates/executor/src/operators/series_labels.rs +++ b/crates/executor/src/operators/series_labels.rs @@ -1108,6 +1108,7 @@ mod columnar_tests { use crate::plan::PhysicalDAG; use crate::runtime::{Limits, Scope}; use planner_types::ir::schema::Field; + use planner_types::stages::Lifetime; /// The row-at-a-time rewrite this operator ran before (the reference). fn reference( @@ -1248,9 +1249,10 @@ mod columnar_tests { ], ) .unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], operator).unwrap(); + dag.add(1, vec![0], operator, Lifetime::None).unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, diff --git a/crates/executor/src/physical_planner/compiled.rs b/crates/executor/src/physical_planner/compiled.rs index 7d074d659..198c997be 100644 --- a/crates/executor/src/physical_planner/compiled.rs +++ b/crates/executor/src/physical_planner/compiled.rs @@ -2,6 +2,7 @@ use super::*; use crate::sources::{DataStore, Scan}; use planner_types::plan::materialized_view::MaterializedView; +use planner_types::stages::Lifetime; #[derive(Clone)] #[cfg_attr(any(test, feature = "test-support"), derive(serde::Serialize))] @@ -14,6 +15,7 @@ enum Node { Operator { inputs: Vec, operator: Operator, + lifetime: Lifetime, }, } @@ -58,12 +60,12 @@ impl CompiledPhysicalDAG { /// Assemble already-lowered operators; leaves read no data store (for /// example [`Operator::source`] holds its rows). pub fn from_operators( - operators: BTreeMap, Operator)>, + operators: BTreeMap, Operator, Lifetime)>, roots: Vec, ) -> Result { let mut result = Self::new(roots); - for (id, (inputs, operator)) in operators { - result.add(id, inputs, operator)?; + for (id, (inputs, operator, lifetime)) in operators { + result.add(id, inputs, operator, lifetime)?; } result.validate()?; Ok(result) @@ -89,8 +91,16 @@ impl CompiledPhysicalDAG { id: NodeId, inputs: Vec, operator: Operator, + lifetime: Lifetime, ) -> Result<(), Error> { - self.insert(id, Node::Operator { inputs, operator }) + self.insert( + id, + Node::Operator { + inputs, + operator, + lifetime, + }, + ) } fn insert(&mut self, id: NodeId, node: Node) -> Result<(), Error> { if self.nodes.insert(id, node).is_some() { @@ -103,7 +113,9 @@ impl CompiledPhysicalDAG { pub fn row_source(&self, id: NodeId) -> Option { match self.nodes.get(&id)? { Node::MaterializedView(_) | Node::RawData(_) => Some(id), - Node::Operator { inputs, operator } => { + Node::Operator { + inputs, operator, .. + } => { let index = operator.row_preserving_input()?; self.row_source(*inputs.get(index)?) } @@ -160,11 +172,13 @@ impl CompiledPhysicalDAG { match node { Node::MaterializedView(MaterializedViewSlot { schema, .. }) | Node::RawData(RawDataSlot { schema, .. }) => { - dag.add(id, vec![], UnboundInput(schema.clone()))? - } - Node::Operator { inputs, operator } => { - dag.add(id, inputs.clone(), operator.clone())? + dag.add(id, vec![], UnboundInput(schema.clone()), SLOT_LIFETIME)? } + Node::Operator { + inputs, + operator, + lifetime, + } => dag.add(id, inputs.clone(), operator.clone(), *lifetime)?, } } dag.validate(&self.roots)?; @@ -188,7 +202,7 @@ impl CompiledPhysicalDAG { slot.schema.clone(), slot.read_window, ); - dag.add(id, vec![], scan)?; + dag.add(id, vec![], scan, SLOT_LIFETIME)?; } Node::RawData(slot) => { let scan = Scan::raw_data( @@ -198,17 +212,24 @@ impl CompiledPhysicalDAG { slot.predicates.clone(), slot.read_window, ); - dag.add(id, vec![], scan)?; - } - Node::Operator { inputs, operator } => { - dag.add(id, inputs.clone(), operator.clone())?; + dag.add(id, vec![], scan, SLOT_LIFETIME)?; } + Node::Operator { + inputs, + operator, + lifetime, + } => dag.add(id, inputs.clone(), operator.clone(), *lifetime)?, } } dag.validate(&self.roots)?; Ok(dag) } } +/// An input slot's lifetime in the runtime DAG. The runtime runs a leaf by +/// the run's scope, not by its lifetime: at query time each consumer reads +/// the store, at ingestion time one read reaches every consumer. +const SLOT_LIFETIME: Lifetime = Lifetime::None; + /// Stands in for an input slot when validating or deriving properties. struct UnboundInput(SchemaRef); impl PhysicalOperator for UnboundInput { diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 0123a59da..651e9b515 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -21,6 +21,7 @@ use planner_types::plan::physical_asap_dag::{ PhysicalASAPOperatorPayload as Payload, }; use planner_types::plan::TimeWindow; +use planner_types::stages::Lifetime; use planner_types::summary_type::{ SketchParams, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, }; @@ -261,6 +262,7 @@ fn compile_internal( auxiliary, inputs, Operator::union(schemas[0].clone(), schemas.len())?, + Lifetime::None, )?; inputs = vec![auxiliary]; schemas.truncate(1); @@ -358,6 +360,7 @@ fn compile_internal( inputs, Operator::current_series(input.clone(), identity, coordinate, value, lookback)? .with_output_schema(output)?, + node.lifetime, )?; continue; } @@ -391,10 +394,15 @@ fn compile_internal( let mut inputs = inputs; for (index, operator) in chain.into_iter().enumerate() { let helper = helper_id(id, index as u64)?; - physical_dag.add(helper, inputs, operator)?; + physical_dag.add(helper, inputs, operator, Lifetime::None)?; inputs = vec![helper]; } - physical_dag.add(id, inputs, last.with_output_schema(output)?)?; + physical_dag.add( + id, + inputs, + last.with_output_schema(output)?, + node.lifetime, + )?; continue; }; let groups = spec @@ -407,6 +415,7 @@ fn compile_internal( id, inputs, top_k_sort(&input, value, groups, *k as u64)?.with_output_schema(output)?, + node.lifetime, )?; continue; } @@ -488,8 +497,8 @@ fn compile_internal( .collect(), )?; let projected = project.schema(); - physical_dag.add(auxiliary, inputs, aggregate)?; - physical_dag.add(helper_id(id, 1)?, vec![auxiliary], project)?; + physical_dag.add(auxiliary, inputs, aggregate, Lifetime::None)?; + physical_dag.add(helper_id(id, 1)?, vec![auxiliary], project, Lifetime::None)?; let stamped = Operator::scope_timestamp(projected, output.clone())?; if output .fields @@ -498,14 +507,20 @@ fn compile_internal( { // Range functions drop the metric name, as their // summary evaluations do. - physical_dag.add(helper_id(id, 2)?, vec![helper_id(id, 1)?], stamped)?; + physical_dag.add( + helper_id(id, 2)?, + vec![helper_id(id, 1)?], + stamped, + Lifetime::None, + )?; physical_dag.add( id, vec![helper_id(id, 2)?], Operator::series_without_name(output)?, + node.lifetime, )?; } else { - physical_dag.add(id, vec![helper_id(id, 1)?], stamped)?; + physical_dag.add(id, vec![helper_id(id, 1)?], stamped, node.lifetime)?; } continue; } @@ -637,12 +652,13 @@ fn compile_internal( .ok_or_else(|| invalid("a chain's build drops its window end")) }) .transpose()?; - physical_dag.add(auxiliary, inputs, window)?; - physical_dag.add(helper_id(id, 1)?, vec![auxiliary], build)?; + physical_dag.add(auxiliary, inputs, window, Lifetime::None)?; + physical_dag.add(helper_id(id, 1)?, vec![auxiliary], build, Lifetime::None)?; physical_dag.add( id, vec![helper_id(id, 1)?], Operator::window_end_timestamp(compact, output, window_end)?, + node.lifetime, )?; continue; } @@ -665,6 +681,7 @@ fn compile_internal( inputs, Operator::evaluate_at(input.clone(), *at, *offset_ms)? .with_output_schema(output)?, + node.lifetime, )?; continue; } @@ -733,11 +750,12 @@ fn compile_internal( groups, )?; let compact = build.schema(); - physical_dag.add(auxiliary, inputs, build)?; + physical_dag.add(auxiliary, inputs, build, Lifetime::None)?; physical_dag.add( id, vec![auxiliary], Operator::scope_timestamp(compact, output)?, + node.lifetime, )?; continue; } @@ -770,7 +788,12 @@ fn compile_internal( [false, false], ) .map_err(|error| invalid(format!("node {id}: {error}")))?; - physical_dag.add(id, inputs, binary.with_output_schema(output)?)?; + physical_dag.add( + id, + inputs, + binary.with_output_schema(output)?, + node.lifetime, + )?; continue; } } @@ -802,16 +825,22 @@ fn compile_internal( .collect(); let project = Operator::project(actual, columns)?.with_output_schema(output.clone())?; - physical_dag.add(auxiliary, inputs, evaluation)?; + physical_dag.add(auxiliary, inputs, evaluation, Lifetime::None)?; if temporal_evaluation_drops_name(node) { - physical_dag.add(auxiliary - 1, vec![auxiliary], project)?; + physical_dag.add( + auxiliary - 1, + vec![auxiliary], + project, + Lifetime::None, + )?; physical_dag.add( id, vec![auxiliary - 1], Operator::series_without_name(output)?, + node.lifetime, )?; } else { - physical_dag.add(id, vec![auxiliary], project)?; + physical_dag.add(id, vec![auxiliary], project, node.lifetime)?; } continue; } @@ -828,11 +857,12 @@ fn compile_internal( // as a per-series `SummaryAgg` does. let merged = bind_operation(node, &schemas)?; let compact = merged.schema(); - physical_dag.add(helper_id(id, 1)?, inputs, merged)?; + physical_dag.add(helper_id(id, 1)?, inputs, merged, Lifetime::None)?; physical_dag.add( id, vec![helper_id(id, 1)?], Operator::scope_timestamp(compact, output)?, + node.lifetime, )?; continue; } @@ -850,7 +880,7 @@ fn compile_internal( let operator = compile_node(node, &schemas) .and_then(|merge| merge.with_emit_empty_global(parameters)) .map_err(|error| invalid(format!("node {id}: {error}")))?; - physical_dag.add(id, inputs, operator)?; + physical_dag.add(id, inputs, operator, node.lifetime)?; continue; } } @@ -859,7 +889,12 @@ fn compile_internal( if let Some(operator) = time_window_operator(id, &graph, &schemas) .map_err(|error| invalid(format!("node {id}: {error}")))? { - physical_dag.add(id, inputs, operator.with_output_schema(output)?)?; + physical_dag.add( + id, + inputs, + operator.with_output_schema(output)?, + node.lifetime, + )?; continue; } let mut operator = compile_node(node, &schemas) @@ -870,10 +905,15 @@ fn compile_internal( } } if temporal_evaluation_drops_name(node) { - physical_dag.add(auxiliary, inputs, operator)?; - physical_dag.add(id, vec![auxiliary], Operator::series_without_name(output)?)?; + physical_dag.add(auxiliary, inputs, operator, Lifetime::None)?; + physical_dag.add( + id, + vec![auxiliary], + Operator::series_without_name(output)?, + node.lifetime, + )?; } else { - physical_dag.add(id, inputs, operator)?; + physical_dag.add(id, inputs, operator, node.lifetime)?; } } } diff --git a/crates/executor/src/physical_planner/promql_values.rs b/crates/executor/src/physical_planner/promql_values.rs index 12a79a4d6..a60938078 100644 --- a/crates/executor/src/physical_planner/promql_values.rs +++ b/crates/executor/src/physical_planner/promql_values.rs @@ -18,5 +18,8 @@ pub fn compile_scalar(value: f64) -> Result { planner_types::ir::schema::DataType::Float64, )? .with_output_schema(scalar_schema())?; - CompiledPhysicalDAG::from_operators(BTreeMap::from([(0, (vec![], operator))]), vec![0]) + CompiledPhysicalDAG::from_operators( + BTreeMap::from([(0, (vec![], operator, Lifetime::None))]), + vec![0], + ) } diff --git a/crates/executor/src/plan/mod.rs b/crates/executor/src/plan/mod.rs index 1b87df0af..4cbeb084b 100644 --- a/crates/executor/src/plan/mod.rs +++ b/crates/executor/src/plan/mod.rs @@ -1,8 +1,9 @@ //! Immutable physical DAG, operator contracts and pre-execution validation. use crate::{ - runtime::{Input, OutputStream, RunContext}, + runtime::{IngestionOutputs, Input, OutputStream, RunContext}, Error, }; +use planner_types::stages::Lifetime; use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, @@ -47,6 +48,9 @@ pub trait PhysicalOperator { pub(crate) struct Node<'a, V, S> { pub(crate) inputs: Vec, pub(crate) operator: Box + 'a>, + /// How long its output is kept, from the plan: what the runtime runs it + /// by (see the crate README). + pub(crate) lifetime: Lifetime, } pub struct PhysicalDAG<'a, V, S> { pub(crate) nodes: BTreeMap>, @@ -64,19 +68,28 @@ impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { id: NodeId, inputs: Vec, operator: impl PhysicalOperator + 'a, + lifetime: Lifetime, ) -> Result<(), Error> { - self.add_boxed(id, inputs, Box::new(operator)) + self.add_boxed(id, inputs, Box::new(operator), lifetime) } pub fn add_boxed( &mut self, id: NodeId, inputs: Vec, operator: Box + 'a>, + lifetime: Lifetime, ) -> Result<(), Error> { if self.nodes.contains_key(&id) { return Err(Error::Invalid(format!("duplicate node {id}"))); } - self.nodes.insert(id, Node { inputs, operator }); + self.nodes.insert( + id, + Node { + inputs, + operator, + lifetime, + }, + ); Ok(()) } pub fn validate(&self, roots: &[NodeId]) -> Result<(), Error> { @@ -153,6 +166,8 @@ impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { .map(|(id, (_, properties))| (id, properties)) .collect()) } + /// A query run: one output stream per root, which the caller may drain + /// in any order. pub fn execute<'r>( &'r self, roots: &[NodeId], @@ -163,4 +178,16 @@ impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { { crate::runtime::execute(self, roots, context) } + /// An ingestion run: one future that resolves to every root's batches, + /// in `roots` order, for the caller to write. + pub fn execute_ingestion<'r>( + &'r self, + roots: &[NodeId], + context: RunContext, + ) -> Result> + 'r, Error> + where + 'a: 'r, + { + crate::runtime::execute_ingestion(self, roots, context) + } } diff --git a/crates/executor/src/runtime/context.rs b/crates/executor/src/runtime/context.rs index e20b7307c..95e361db6 100644 --- a/crates/executor/src/runtime/context.rs +++ b/crates/executor/src/runtime/context.rs @@ -17,13 +17,11 @@ pub enum Scope { } #[derive(Clone, Debug)] pub struct Limits { - pub max_buffered_batches: usize, pub max_bytes: usize, } impl Default for Limits { fn default() -> Self { Self { - max_buffered_batches: 8, max_bytes: 64 * 1024 * 1024, } } @@ -43,7 +41,7 @@ pub struct RunContext { } impl RunContext { pub fn new(scope: Scope, limits: Limits) -> Result { - if limits.max_buffered_batches == 0 || limits.max_bytes == 0 { + if limits.max_bytes == 0 { return Err(Error::Invalid("execution limits must be positive".into())); } if matches!(&scope, Scope::Ingestion { window_start_ms, window_end_ms, .. } if window_start_ms > window_end_ms) diff --git a/crates/executor/src/runtime/mod.rs b/crates/executor/src/runtime/mod.rs index 89039cffc..8f8b08f5e 100644 --- a/crates/executor/src/runtime/mod.rs +++ b/crates/executor/src/runtime/mod.rs @@ -1,9 +1,10 @@ -//! Per-run producer sharing, streams, backpressure and resource ownership. +//! Per-run producers run by each node's `Lifetime`, streams and resource ownership. use crate::{ - plan::{NodeId, PhysicalDAG}, + plan::{Node, NodeId, PhysicalDAG}, Error, }; -use futures::{stream::LocalBoxStream, Stream}; +use futures::{stream::LocalBoxStream, Stream, TryStreamExt}; +use planner_types::stages::Lifetime; use std::{ cell::RefCell, collections::{BTreeMap, VecDeque}, @@ -16,6 +17,8 @@ use std::{ mod context; pub use context::{Limits, Reservation, RunContext, Scope}; pub type OutputStream<'a, V> = LocalBoxStream<'a, Result>; +/// Every root's batches of an ingestion run, in the order of its roots. +pub type IngestionOutputs = Result>>, Error>; /// An output owns its memory reservation even after it leaves the DAG's queue. pub struct SharedValue { value: Arc, @@ -41,10 +44,56 @@ impl SharedValue { } } +/// A query-time run: one output stream per root. Fails before any operator +/// starts when a computed node that is not materialized +/// ([`Lifetime::None`]) has more than one consumer. pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( dag: &'r PhysicalDAG<'_, V, S>, roots: &[NodeId], context: RunContext, +) -> Result>, Error> { + if context.evaluation_time_ms().is_none() { + return Err(Error::Invalid( + "an ingestion run is executed by `execute_ingestion`".into(), + )); + } + start(dag, roots, context) +} + +/// An ingestion run: every root's batches, the outputs the caller writes. +/// The runtime polls the roots together, so a node feeding several of them +/// is computed once and each batch reaches every consumer. +pub(crate) fn execute_ingestion<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( + dag: &'r PhysicalDAG<'_, V, S>, + roots: &[NodeId], + context: RunContext, +) -> Result> + 'r, Error> { + if context.evaluation_time_ms().is_some() { + return Err(Error::Invalid( + "a query run is executed by `execute`".into(), + )); + } + let outputs = start(dag, roots, context)?; + Ok(futures::future::try_join_all( + outputs.into_iter().map(TryStreamExt::try_collect), + )) +} + +/// A node runs once per time it is evaluated at: an `@` anchor above it +/// evaluates it at another time than the run's. +type ProducerKey = (NodeId, Option); + +/// Whether each consumer of `node` reads it on its own: at query time, a +/// leaf (a store read, a raw data or materialized view slot) is read once +/// per consumer, as the planner charges a not-materialized read per edge. +fn read_per_consumer(node: &Node<'_, V, S>, context: &RunContext) -> bool { + node.inputs.is_empty() && context.evaluation_time_ms().is_some() +} + +fn start<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( + dag: &'r PhysicalDAG<'_, V, S>, + roots: &[NodeId], + context: RunContext, ) -> Result>, Error> { if context.is_cancelled() { return Err(Error::Cancelled); @@ -59,9 +108,55 @@ pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( pending.extend(node.inputs.iter().copied()); } } - // A node runs once per time it is evaluated at: an `@` anchor above it - // evaluates it at another time than the run's. - type ProducerKey = (NodeId, Option); + fn input_context( + dag: &PhysicalDAG<'_, V, S>, + id: NodeId, + context: &RunContext, + ) -> Result { + let node = &dag.nodes[&id]; + node.operator + .input_context(context) + .map_err(|source| Error::AtNode { + node: id, + operation: node.operator.name().into(), + source: Box::new(source), + }) + } + // The consumers of each shared producer, counted before any operator + // starts. + fn count_consumers( + dag: &PhysicalDAG<'_, V, S>, + id: NodeId, + context: &RunContext, + consumers: &mut BTreeMap, + ) -> Result<(), Error> { + let count = consumers + .entry((id, context.evaluation_time_ms())) + .or_default(); + *count += 1; + if *count == 1 { + let input_context = input_context(dag, id, context)?; + for &child in &dag.nodes[&id].inputs { + count_consumers(dag, child, &input_context, consumers)?; + } + } + Ok(()) + } + let mut consumers = BTreeMap::new(); + for &root in roots { + count_consumers(dag, root, &context, &mut consumers)?; + } + if context.evaluation_time_ms().is_some() { + for (&(id, _), &count) in &consumers { + let node = &dag.nodes[&id]; + if count > 1 && !node.inputs.is_empty() && node.lifetime == Lifetime::None { + return Err(Error::Invalid(format!( + "node {id} ({}) is not materialized but has {count} consumers at query time", + node.operator.name() + ))); + } + } + } fn build<'r, V: 'r, S: 'r>( dag: &'r PhysicalDAG<'_, V, S>, id: NodeId, @@ -69,18 +164,12 @@ pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( states: &mut BTreeMap>>>, ) -> Result>>, Error> { let key = (id, context.evaluation_time_ms()); - if let Some(state) = states.get(&key) { + let node = &dag.nodes[&id]; + let shared = !read_per_consumer(node, context); + if let Some(state) = states.get(&key).filter(|_| shared) { return Ok(Rc::clone(state)); } - let node = &dag.nodes[&id]; - let input_context = - node.operator - .input_context(context) - .map_err(|source| Error::AtNode { - node: id, - operation: node.operator.name().into(), - source: Box::new(source), - })?; + let input_context = input_context(dag, id, context)?; let mut inputs = Vec::new(); for &child in &node.inputs { inputs.push(Input::subscribe(build(dag, child, &input_context, states)?)); @@ -100,16 +189,17 @@ pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( operation: node.operator.name().into(), size: Box::new(move |value| op.output_bytes(value)), context: context.clone(), - queue: VecDeque::new(), - base: 0, + hold_until_finished: node.lifetime != Lifetime::None && !node.inputs.is_empty(), + unread: BTreeMap::new(), next_reader: 0, batches_polled: 0, - readers: BTreeMap::new(), waiters: BTreeMap::new(), finished: false, failure: None, })); - states.insert(key, Rc::clone(&state)); + if shared { + states.insert(key, Rc::clone(&state)); + } Ok(state) } let mut states = BTreeMap::new(); @@ -118,40 +208,38 @@ pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( .map(|&id| build(dag, id, &context, &mut states).map(Input::subscribe)) .collect() } +/// One run of a node, read by every consumer it has in the run. struct Producer<'a, V> { node: NodeId, operation: String, stream: Option>, size: Box usize + 'a>, context: RunContext, - queue: VecDeque>, - base: u64, + /// A materialized node ([`Lifetime::OneEvaluation`] or + /// [`Lifetime::AcrossEvaluations`]) is computed to the end before any + /// consumer reads it, and its batches stay charged to the run until + /// every consumer has read them. + hold_until_finished: bool, + /// The batches each reader has not read yet, by reader. + unread: BTreeMap>>, next_reader: u64, batches_polled: usize, - readers: BTreeMap, + /// The readers waiting for the next batch or the end. waiters: BTreeMap, finished: bool, failure: Option, } impl Producer<'_, V> { - fn trim(&mut self) { - let minimum = self - .readers - .values() - .copied() - .min() - .unwrap_or(self.base + self.queue.len() as u64); - while self.base < minimum { - self.queue.pop_front(); - self.base += 1; - } + fn wake_all(&mut self) { for (_, waker) in std::mem::take(&mut self.waiters) { waker.wake(); } - if self.readers.is_empty() { - self.stream = None; - self.queue.clear(); - } + } + fn finish(&mut self, failure: Option) { + self.failure = failure; + self.finished = true; + self.stream = None; + self.wake_all(); } } pub struct Input<'a, V> { @@ -165,8 +253,7 @@ impl<'a, V> Input<'a, V> { let mut state = producer.borrow_mut(); let id = state.next_reader; state.next_reader += 1; - let base = state.base; - state.readers.insert(id, base); + state.unread.insert(id, VecDeque::new()); id }; Self { @@ -179,9 +266,11 @@ impl<'a, V> Input<'a, V> { impl Drop for Input<'_, V> { fn drop(&mut self) { let mut state = self.producer.borrow_mut(); - state.readers.remove(&self.reader); + state.unread.remove(&self.reader); state.waiters.remove(&self.reader); - state.trim(); + if state.unread.is_empty() { + state.stream = None; + } } } impl Stream for Input<'_, V> { @@ -193,84 +282,77 @@ impl Stream for Input<'_, V> { } let mut state = this.producer.borrow_mut(); state.context.register(cx.waker()); - if state.context.is_cancelled() { - state.failure = Some(Error::Cancelled); - state.finished = true; - state.stream = None; - state.queue.clear(); - } - let position = state.readers[&this.reader]; - let index = (position - state.base) as usize; - if let Some(value) = state.queue.get(index).cloned() { - state.readers.insert(this.reader, position + 1); - state.trim(); - return Poll::Ready(Some(Ok(value))); - } - if state.finished { - this.done = true; - state.readers.remove(&this.reader); - let failure = state.failure.clone(); - state.trim(); - return Poll::Ready(failure.map(Err)); - } - state.waiters.insert(this.reader, cx.waker().clone()); - if state.queue.len() >= state.context.control.limits.max_buffered_batches { - return Poll::Pending; - } - // Always-ready sources must still give cancellation and other roots a turn. - if state.batches_polled >= 32 { - state.batches_polled = 0; - cx.waker().wake_by_ref(); - return Poll::Pending; - } - let polled = state - .stream - .as_mut() - .expect("unfinished producer") - .as_mut() - .poll_next(cx); - if matches!(&polled, Poll::Ready(Some(Ok(_)))) { - state.batches_polled += 1; - } - match polled { - Poll::Pending => Poll::Pending, - Poll::Ready(Some(Ok(value))) => match state.context.reserve((state.size)(&value)) { - Ok(reservation) => { - let value = SharedValue { - value: Arc::new(value), - _reservation: Rc::new(reservation), - }; - state.queue.push_back(value.clone()); - state.readers.insert(this.reader, position + 1); - state.trim(); - Poll::Ready(Some(Ok(value))) + loop { + if state.context.is_cancelled() && !state.finished { + state.finish(Some(Error::Cancelled)); + for unread in state.unread.values_mut() { + unread.clear(); } - Err(error) => { - state.failure = Some(error.clone()); - state.finished = true; - state.stream = None; - this.done = true; - state.readers.remove(&this.reader); - state.trim(); - Poll::Ready(Some(Err(error))) + } + if state.finished || !state.hold_until_finished { + let unread = state + .unread + .get_mut(&this.reader) + .expect("subscribed reader"); + if let Some(value) = unread.pop_front() { + return Poll::Ready(Some(Ok(value))); } - }, - Poll::Ready(result) => { - let error = result.and_then(Result::err).map(|source| match source { - Error::AtNode { .. } | Error::Cancelled | Error::MemoryLimit => source, - source => Error::AtNode { - node: state.node, - operation: state.operation.clone(), - source: Box::new(source), - }, - }); - state.failure = error.clone(); - state.finished = true; - state.stream = None; + } + if state.finished { this.done = true; - state.readers.remove(&this.reader); - state.trim(); - Poll::Ready(error.map(Err)) + state.unread.remove(&this.reader); + return Poll::Ready(state.failure.clone().map(Err)); + } + // Always-ready sources must still give cancellation and other roots a turn. + if state.batches_polled >= 32 { + state.batches_polled = 0; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + let polled = state + .stream + .as_mut() + .expect("unfinished producer") + .as_mut() + .poll_next(cx); + match polled { + Poll::Pending => { + // Any reader's progress wakes this one: the upstream + // wakes only the reader that polled it last. + state.waiters.insert(this.reader, cx.waker().clone()); + return Poll::Pending; + } + Poll::Ready(Some(Ok(value))) => { + state.batches_polled += 1; + match state.context.reserve((state.size)(&value)) { + Ok(reservation) => { + let value = SharedValue { + value: Arc::new(value), + _reservation: Rc::new(reservation), + }; + for unread in state.unread.values_mut() { + unread.push_back(value.clone()); + } + // A held output is read only once it has + // finished, which wakes every reader. + if !state.hold_until_finished { + state.wake_all(); + } + } + Err(error) => state.finish(Some(error)), + } + } + Poll::Ready(result) => { + let error = result.and_then(Result::err).map(|source| match source { + Error::AtNode { .. } | Error::Cancelled | Error::MemoryLimit => source, + source => Error::AtNode { + node: state.node, + operation: state.operation.clone(), + source: Box::new(source), + }, + }); + state.finish(error); + } } } } diff --git a/crates/executor/src/runtime/tests.rs b/crates/executor/src/runtime/tests.rs index 20816e952..a10864712 100644 --- a/crates/executor/src/runtime/tests.rs +++ b/crates/executor/src/runtime/tests.rs @@ -1,6 +1,7 @@ use super::*; use crate::plan::PhysicalOperator; use futures::{executor::block_on, stream, StreamExt}; +use planner_types::stages::StorageMedium; use std::cell::Cell; struct Source { @@ -66,10 +67,7 @@ fn context() -> RunContext { Scope::Query { evaluation_time_ms: 100, }, - Limits { - max_buffered_batches: 1, - max_bytes: 1024, - }, + Limits { max_bytes: 1024 }, ) .unwrap() } @@ -88,71 +86,67 @@ fn source(fail: bool) -> (Source, Rc>, Rc>) { ) } -// A shared producer runs once, and the slow reader bounds producer progress. +const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); + +// At query time each consumer of a store read (a leaf) reads it on its own, +// so one consumer never waits for another. #[test] -fn shared_source_backpressure_and_reader_drop() { +fn query_time_leaves_are_read_per_consumer() { let (source, starts, polls) = source(false); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], source).unwrap(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); let context = context(); let mut readers = dag.execute(&[0, 0], context.clone()).unwrap(); - let mut slow = readers.pop().unwrap(); - let mut fast = readers.pop().unwrap(); - assert_eq!(starts.get(), 1); - let first = block_on(fast.next()).unwrap().unwrap(); - assert_eq!(*first, 0); - let mut cx = Context::from_waker(futures::task::noop_waker_ref()); - assert!(Pin::new(&mut fast).poll_next(&mut cx).is_pending()); - assert_eq!(polls.get(), 1); - let same = block_on(slow.next()).unwrap().unwrap(); - assert!(Arc::ptr_eq(&first.value, &same.value)); - drop(same); - drop(first); - assert_eq!(context.retained_bytes(), 0); - assert_eq!(*block_on(fast.next()).unwrap().unwrap(), 1); - drop(slow); - assert_eq!(*block_on(fast.next()).unwrap().unwrap(), 2); - assert_eq!(*block_on(fast.next()).unwrap().unwrap(), 3); - assert!(block_on(fast.next()).is_none()); + let second = readers.pop().unwrap(); + let mut first = readers.pop().unwrap(); + assert_eq!(starts.get(), 2); + assert_eq!(*block_on(first.next()).unwrap().unwrap(), 0); + drop(second); + for expected in 1..4 { + assert_eq!(*block_on(first.next()).unwrap().unwrap(), expected); + } + assert!(block_on(first.next()).is_none()); assert_eq!(polls.get(), 4); - drop(fast); + drop(first); assert_eq!(context.retained_bytes(), 0); } -// Independent branches consume a common node concurrently without duplicate work. +// A materialized node runs once per run, to the end, and every consumer +// reads its finished output, charged to the run until all have read it. #[test] -fn diamond_and_run_isolation() { +fn materialized_node_runs_once_and_is_held() { let (source, starts, polls) = source(false); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], source).unwrap(); - dag.add(1, vec![0], Identity).unwrap(); - dag.add(2, vec![0], Identity).unwrap(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); + dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); + dag.add(3, vec![1], Identity, Lifetime::None).unwrap(); for _ in 0..2 { - let mut outputs = dag.execute(&[1, 2], context()).unwrap(); - let a = outputs.pop().unwrap(); - let b = outputs.pop().unwrap(); - let (a, b) = - block_on(async { futures::join!(a.collect::>(), b.collect::>()) }); - assert_eq!( - a.iter().map(|v| **v.as_ref().unwrap()).collect::>(), - vec![0, 1, 2, 3] - ); - assert_eq!( - b.iter().map(|v| **v.as_ref().unwrap()).collect::>(), - vec![0, 1, 2, 3] - ); + let run = context(); + let outputs = dag.execute(&[2, 3], run.clone()).unwrap(); + for output in outputs { + assert_eq!( + values(block_on(output.collect::>())), + vec![Ok(0), Ok(1), Ok(2), Ok(3)] + ); + } + // The held output, and the first root's batches collected while the + // second root has not read it yet. + assert_eq!(run.peak_bytes(), 2 * 4 * 8); + assert_eq!(run.retained_bytes(), 0); } assert_eq!(starts.get(), 2); assert_eq!(polls.get(), 8); } -// Failure reaches every subscriber; cancellation stops further producer work. +// Failure reaches every consumer; cancellation stops further producer work. #[test] fn broadcast_error_and_cancel() { let (source, _, polls) = source(true); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], source).unwrap(); - let mut outputs = dag.execute(&[0, 0], context()).unwrap(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); + let mut outputs = dag.execute(&[1, 1], context()).unwrap(); let a = outputs.pop().unwrap(); let b = outputs.pop().unwrap(); let (a, b) = block_on(async { futures::join!(a.collect::>(), b.collect::>()) }); @@ -172,22 +166,13 @@ fn broadcast_error_and_cancel() { assert_eq!(polls.get(), 2); } -// Retaining a consumer output retains its budget lease after queue eviction. +// Retaining a consumer output retains its budget lease after it is read. #[test] fn retained_outputs_count_against_budget() { let (source, _, _) = source(false); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], source).unwrap(); - let run = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - Limits { - max_buffered_batches: 1, - max_bytes: 8, - }, - ) - .unwrap(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + let run = limits_context(8); let mut input = dag.execute(&[0], run.clone()).unwrap().remove(0); let held = block_on(input.next()).unwrap().unwrap(); assert_eq!(run.retained_bytes(), 8); @@ -206,16 +191,16 @@ fn retained_outputs_count_against_budget() { fn invalid_dags_do_not_start_sources() { let (source, starts, _) = source(false); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], source).unwrap(); - dag.add(1, vec![2], Identity).unwrap(); - dag.add(2, vec![1], Identity).unwrap(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![2], Identity, Lifetime::None).unwrap(); + dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); assert!(dag.execute(&[0, 1], context()).is_err()); assert_eq!(starts.get(), 0); let mut missing = PhysicalDAG::default(); - missing.add(1, vec![9], Identity).unwrap(); + missing.add(1, vec![9], Identity, Lifetime::None).unwrap(); assert!(missing.validate(&[1]).is_err()); let mut arity = PhysicalDAG::default(); - arity.add(1, vec![], Identity).unwrap(); + arity.add(1, vec![], Identity, Lifetime::None).unwrap(); assert!(arity.validate(&[1]).is_err()); } @@ -225,7 +210,7 @@ fn ready_sources_cooperate_with_cancellation() { let (mut source, _, polls) = source(false); source.end = 10_000; let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], source).unwrap(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); let context = context(); let mut input = dag.execute(&[0], context.clone()).unwrap().remove(0); block_on(async { @@ -252,9 +237,223 @@ fn ready_sources_cooperate_with_cancellation() { fn depth_limit_covers_shared_paths() { let (source, _, _) = source(false); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], source).unwrap(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); for id in 1..129 { - dag.add(id, vec![id - 1], Identity).unwrap(); + dag.add(id, vec![id - 1], Identity, Lifetime::None).unwrap(); } assert!(dag.validate(&(0..129).collect::>()).is_err()); } + +// Reads its left input to the end before its right, as `SeriesBinary` does. +struct LeftThenRight; +impl PhysicalOperator for LeftThenRight { + fn name(&self) -> &str { + "LeftThenRight" + } + fn input_schemas(&self) -> Vec<()> { + vec![(), ()] + } + fn output_schema(&self) {} + fn output_bytes(&self, _: &u64) -> usize { + 8 + } + fn start<'a>( + &'a self, + inputs: Vec>, + _: RunContext, + ) -> Result, Error> { + Ok(stream::iter(inputs) + .flatten() + .map(|value| value.map(|v| *v)) + .boxed_local()) + } +} + +// Reads its input one minute before the run's evaluation time, like a +// `TimeShift` with an `@` anchor. +struct MinuteEarlier; +impl PhysicalOperator for MinuteEarlier { + fn name(&self) -> &str { + "MinuteEarlier" + } + fn input_context(&self, context: &RunContext) -> Result { + Ok(context.at_evaluation_time(context.evaluation_time_ms().unwrap() - 60_000)) + } + fn input_schemas(&self) -> Vec<()> { + vec![()] + } + fn output_schema(&self) {} + fn output_bytes(&self, _: &u64) -> usize { + 8 + } + fn start<'a>( + &'a self, + mut inputs: Vec>, + _: RunContext, + ) -> Result, Error> { + Ok(inputs + .remove(0) + .map(|value| value.map(|v| *v)) + .boxed_local()) + } +} + +/// Runs `run` on its own thread and fails the test if it does not return in +/// time, so a deadlocked run fails instead of hanging the test binary. +fn within_timeout(run: impl FnOnce() -> T + Send + 'static) -> T { + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || sender.send(run()).unwrap()); + receiver + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("the run did not finish") +} + +/// More batches than any queue bound the runtime ever had (8 by default). +const MANY_BATCHES: u64 = 20; +/// Node 0 emits `MANY_BATCHES` batches; nodes 1 and 2 each read it. +fn two_readers_of_one_leaf(dag: &mut PhysicalDAG<'static, u64, ()>) -> Rc> { + let (mut source, starts, _) = source(false); + source.end = MANY_BATCHES; + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], Identity, Lifetime::None).unwrap(); + dag.add(2, vec![0], Identity, Lifetime::None).unwrap(); + starts +} +fn limits_context(max_bytes: usize) -> RunContext { + RunContext::new( + Scope::Query { + evaluation_time_ms: 0, + }, + Limits { max_bytes }, + ) + .unwrap() +} +fn ingestion_context(max_bytes: usize) -> RunContext { + RunContext::new( + Scope::Ingestion { + window_start_ms: 0, + window_end_ms: 10, + }, + Limits { max_bytes }, + ) + .unwrap() +} +fn values(results: Vec, Error>>) -> Vec> { + results.into_iter().map(|r| r.map(|v| *v)).collect() +} +fn all_batches() -> Vec> { + (0..MANY_BATCHES).map(Ok).collect() +} + +// Two queries read one stored view (y16): draining their roots one after +// another gives the same answers as polling them together. +#[test] +fn roots_reading_one_leaf_drain_in_order() { + within_timeout(|| { + let mut dag = PhysicalDAG::default(); + two_readers_of_one_leaf(&mut dag); + let outputs = dag.execute(&[1, 2], limits_context(1024)).unwrap(); + let joined: Vec<_> = block_on(futures::future::join_all( + outputs.into_iter().map(|output| output.collect::>()), + )) + .into_iter() + .map(values) + .collect(); + assert_eq!(joined, vec![all_batches(), all_batches()]); + let run = limits_context(1024); + let outputs = dag.execute(&[1, 2], run.clone()).unwrap(); + let drained: Vec<_> = outputs + .into_iter() + .map(|output| values(block_on(output.collect::>()))) + .collect(); + assert_eq!(drained, joined); + assert_eq!(run.retained_bytes(), 0); + }); +} + +// One query whose two branches read one stored view (b17, g10), combined by +// an operator that reads its left branch to the end first. +#[test] +fn one_root_with_two_branches_over_one_leaf() { + within_timeout(|| { + let mut dag = PhysicalDAG::default(); + two_readers_of_one_leaf(&mut dag); + dag.add(3, vec![1, 2], LeftThenRight, Lifetime::None) + .unwrap(); + let output = dag.execute(&[3], limits_context(1024)).unwrap().remove(0); + let both: Vec<_> = all_batches().into_iter().chain(all_batches()).collect(); + assert_eq!(values(block_on(output.collect::>())), both); + }); +} + +// At ingestion time one read of a source feeds several chain builds: the +// runtime polls every root itself and each batch reaches every consumer. +#[test] +fn ingestion_fan_out_reads_the_source_once() { + within_timeout(|| { + let mut dag = PhysicalDAG::default(); + let starts = two_readers_of_one_leaf(&mut dag); + dag.add(3, vec![0], Identity, Lifetime::None).unwrap(); + dag.add(4, vec![1, 2], LeftThenRight, Lifetime::None) + .unwrap(); + let run = ingestion_context(1024); + let outputs = block_on(dag.execute_ingestion(&[3, 4], run.clone()).unwrap()).unwrap(); + let outputs: Vec<_> = outputs + .into_iter() + .map(|batches| batches.into_iter().map(Ok).collect::>()) + .map(values) + .collect(); + let both: Vec<_> = all_batches().into_iter().chain(all_batches()).collect(); + assert_eq!(outputs, vec![all_batches(), both]); + assert_eq!(starts.get(), 1); + assert_eq!(run.retained_bytes(), 0); + assert!(dag.execute(&[3], ingestion_context(1024)).is_err()); + assert!(dag.execute_ingestion(&[3], limits_context(1024)).is_err()); + }); +} + +// A computed node that is not materialized has exactly one consumer at +// query time; two mean a planner bug, reported before any operator starts. +#[test] +fn query_time_fan_out_of_a_not_materialized_node_is_an_error() { + let (source, starts, _) = source(false); + let mut dag = PhysicalDAG::default(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], Identity, Lifetime::None).unwrap(); + dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); + dag.add(3, vec![1], Identity, Lifetime::None).unwrap(); + let Err(error) = dag.execute(&[2, 3], context()) else { + panic!("a not-materialized node with two consumers ran"); + }; + assert_eq!( + error, + Error::Invalid( + "node 1 (Identity) is not materialized but has 2 consumers at query time".into() + ) + ); + assert_eq!(starts.get(), 0); + // Read at two evaluation times, it runs once per time, each with one consumer. + dag.add(4, vec![1], MinuteEarlier, Lifetime::None).unwrap(); + let outputs = dag.execute(&[2, 4], context()).unwrap(); + for output in outputs { + assert_eq!(values(block_on(output.collect::>())).len(), 4); + } +} + +// A one-evaluation output over the run's byte budget fails with +// `MemoryLimit`; it is not spilled. +#[test] +fn one_evaluation_output_over_budget_is_a_memory_limit() { + within_timeout(|| { + let mut dag = PhysicalDAG::default(); + let (mut source, _, _) = source(false); + source.end = MANY_BATCHES; + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); + let run = limits_context(5 * 8); + let output = dag.execute(&[1], run.clone()).unwrap().remove(0); + let read = values(block_on(output.collect::>())); + assert_eq!(read.last(), Some(&Err(Error::MemoryLimit))); + assert_eq!(run.retained_bytes(), 0); + }); +} diff --git a/crates/executor/tests/aggregate_memory_limit.rs b/crates/executor/tests/aggregate_memory_limit.rs index ac4a75085..d3ece60a5 100644 --- a/crates/executor/tests/aggregate_memory_limit.rs +++ b/crates/executor/tests/aggregate_memory_limit.rs @@ -9,6 +9,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::Error; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -87,12 +88,18 @@ fn run_aggregate( }) .collect(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input.clone(), batches).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), batches).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 1, vec![0], Operator::aggregate(input, vec![0], measures).unwrap(), + Lifetime::None, ) .unwrap(); let context = RunContext::new( @@ -101,7 +108,6 @@ fn run_aggregate( }, Limits { max_bytes: limit_bytes, - ..Limits::default() }, ) .unwrap(); diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index b7f7409c8..3e0fc952b 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -12,6 +12,7 @@ use planner_types::ir::scalar::ScalarValue; use planner_types::ir::schema::{DataType, Field, FieldDataType}; use planner_types::ir::Predicate; use planner_types::ir::ScalarExpr; +use planner_types::stages::Lifetime; use std::sync::Arc; fn schema(width: usize) -> SchemaRef { @@ -33,10 +34,7 @@ fn context(max_bytes: usize) -> RunContext { Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes, - ..Limits::default() - }, + Limits { max_bytes }, ) .unwrap() } @@ -50,6 +48,7 @@ fn source(n: usize) -> PhysicalDAG<'static, Batch, SchemaRef> { vec![Batch::try_new(schema(1), vec![vec![Value::Int64(1)]; n]).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); dag @@ -222,8 +221,13 @@ fn cooperative_sort_preserves_ties_across_chunks() { ) .unwrap(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(schema(2), vec![batch]).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(schema(2), vec![batch]).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 1, vec![0], @@ -238,6 +242,7 @@ fn cooperative_sort_preserves_ties_across_chunks() { None, ) .unwrap(), + Lifetime::None, ) .unwrap(); let mut output = dag @@ -289,6 +294,7 @@ fn weighted_summary_build_yields_within_a_batch() { 0, vec![], Operator::source(input.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); @@ -338,6 +344,7 @@ fn frequency_dictionary_enforces_memory_budget() { .unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); for (reduction, succeeds) in [ diff --git a/crates/executor/tests/cheap_measures_memory_limit.rs b/crates/executor/tests/cheap_measures_memory_limit.rs index 873286fef..997d96c7a 100644 --- a/crates/executor/tests/cheap_measures_memory_limit.rs +++ b/crates/executor/tests/cheap_measures_memory_limit.rs @@ -10,6 +10,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::Error; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -47,22 +48,21 @@ fn aggregate( 0, vec![], Operator::source(input.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 1, vec![0], Operator::aggregate(input, groups, measures).unwrap(), + Lifetime::None, ) .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes, - ..Limits::default() - }, + Limits { max_bytes }, ) .unwrap(); let baseline = ALLOCATED.load(Ordering::Relaxed); diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index a3713995e..ababbefbc 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -1,5 +1,8 @@ #![allow(dead_code)] +use asap_executor::runtime::{RunContext, Scope}; +use asap_executor::values::{Batch, SchemaRef}; +use futures::TryStreamExt; use planner_types::ir::{OperatorNode, QueryRoot}; use std::rc::Rc; @@ -51,3 +54,29 @@ fn run_all_stages( ) .expect("Pass 1 accepts the query") } + +/// Every root's batches in one run of `dag`, in `roots` order: through +/// `execute` at query time (each root drained in turn) and +/// `execute_ingestion` at ingestion time. +pub fn run_roots( + dag: &asap_executor::plan::PhysicalDAG<'_, Batch, SchemaRef>, + roots: &[u64], + context: RunContext, +) -> Result>, asap_executor::Error> { + futures::executor::block_on(async { + let outputs = match context.scope { + Scope::Ingestion { .. } => dag.execute_ingestion(roots, context)?.await?, + Scope::Query { .. } => { + let mut outputs = Vec::new(); + for output in dag.execute(roots, context)? { + outputs.push(output.try_collect::>().await?); + } + outputs + } + }; + Ok(outputs + .into_iter() + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .collect()) + }) +} diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 139941149..071eccef3 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -14,6 +14,7 @@ use futures::{executor::block_on, StreamExt}; use planner_types::accuracy::AccuracyTarget; use planner_types::ir::schema::{DataType, *}; use planner_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; +use planner_types::stages::Lifetime; use planner_types::summary_type::*; use std::{collections::BTreeMap, sync::Arc}; @@ -89,12 +90,13 @@ fn input(samples: &[(&str, i64, f64)]) -> Batch { fn snapshot_plan(data: Batch) -> CompiledPhysicalDAG { CompiledPhysicalDAG::from_operators( BTreeMap::from([ - (0, (vec![], source(data))), + (0, (vec![], source(data), Lifetime::None)), ( 1, ( vec![0], Operator::current_series(schema(), 3, 0, 1, 60_000).unwrap(), + Lifetime::None, ), ), ]), @@ -196,16 +198,17 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { let plan = |data: Batch| { CompiledPhysicalDAG::from_operators( BTreeMap::from([ - (0, (vec![], source(data))), + (0, (vec![], source(data), Lifetime::None)), ( 1, ( vec![0], Operator::current_series(schema(), 3, 0, 1, 60_000).unwrap(), + Lifetime::None, ), ), - (2, (vec![1], build.clone())), - (3, (vec![2], read.clone())), + (2, (vec![1], build.clone(), Lifetime::None)), + (3, (vec![2], read.clone(), Lifetime::None)), ]), vec![3], ) @@ -258,7 +261,6 @@ fn current_series_observes_resource_limits() { }, Limits { max_bytes: if cancelled { 1 << 20 } else { 1 }, - ..Limits::default() }, ) .unwrap(); diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index a55052c4b..947fba376 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -8,7 +8,7 @@ use asap_executor::{ sources::InMemoryDataStore, values::{Batch, Value}, }; -use futures::{executor::block_on, StreamExt}; + use planner_types::ir::operator::Source; use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ExprSemantics, ScalarValue}; @@ -110,22 +110,17 @@ fn kept_users(scope: Scope) -> Result, String> { .instantiate(Arc::new(data_store())) .unwrap(); let context = RunContext::new(scope, Limits::default()).unwrap(); - block_on(async { - let mut output = bound - .execute(&[0], context) - .map_err(|e| e.to_string())? - .remove(0); - let mut users = Vec::new(); - while let Some(batch) = output.next().await { - for row in batch.map_err(|e| e.to_string())?.to_rows() { - let Value::Int64(user) = row[1] else { - panic!("user {:?}", row[1]) - }; - users.push(user); - } - } - Ok(users) - }) + let batches = crate::common::run_roots(&bound, &[0], context) + .map_err(|e| e.to_string())? + .remove(0); + let mut users = Vec::new(); + for row in batches.iter().flat_map(Batch::to_rows) { + let Value::Int64(user) = row[1] else { + panic!("user {:?}", row[1]) + }; + users.push(user); + } + Ok(users) } #[test] diff --git a/crates/executor/tests/exact_state_columns.rs b/crates/executor/tests/exact_state_columns.rs index f8cae953a..caf10dbe9 100644 --- a/crates/executor/tests/exact_state_columns.rs +++ b/crates/executor/tests/exact_state_columns.rs @@ -9,6 +9,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::summary_kernels::exact::ExactEvaluation; use asap_executor::values::{Batch, SchemaRef, Value}; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::sync::Arc; fn field(name: &str, dtype: DataType) -> Field { @@ -66,9 +67,14 @@ fn context() -> RunContext { fn run(operator: Operator, inputs: Vec) -> Vec { let mut dag = PhysicalDAG::default(); let schema = inputs[0].schema().clone(); - dag.add(0, vec![], Operator::source(schema, inputs).unwrap()) - .unwrap(); - dag.add(1, vec![0], operator).unwrap(); + dag.add( + 0, + vec![], + Operator::source(schema, inputs).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add(1, vec![0], operator, Lifetime::None).unwrap(); block_on(async { let mut output = dag.execute(&[1], context()).unwrap().remove(0); let mut batches = Vec::new(); diff --git a/crates/executor/tests/grouped_limit_memory_limit.rs b/crates/executor/tests/grouped_limit_memory_limit.rs index e1b03188a..4d9068176 100644 --- a/crates/executor/tests/grouped_limit_memory_limit.rs +++ b/crates/executor/tests/grouped_limit_memory_limit.rs @@ -9,6 +9,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::Error; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -60,17 +61,22 @@ fn many_groups_in_one_batch_fail_near_the_memory_limit() { 0, vec![], Operator::source(input.clone(), vec![batch]).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add( + 1, + vec![0], + Operator::limit(input, 1, 0, vec![0]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], Operator::limit(input, 1, 0, vec![0]).unwrap()) - .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, Limits { max_bytes: LIMIT_BYTES, - ..Limits::default() }, ) .unwrap(); diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index bc889dd24..e8ecf4f50 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -210,7 +210,14 @@ fn try_run( ) .unwrap(); operators.insert(300, (vec![merged], read)); - let compiled = CompiledPhysicalDAG::from_operators(operators, vec![300]).unwrap(); + let compiled = CompiledPhysicalDAG::from_operators( + operators + .into_iter() + .map(|(id, (inputs, operator))| (id, (inputs, operator, Lifetime::None))) + .collect(), + vec![300], + ) + .unwrap(); let dag = compiled .instantiate(Arc::new(InMemoryDataStore::default())) .unwrap(); diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index cdf195bb8..fe1873d52 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -91,10 +91,15 @@ fn top_items(build: Operator, input: SchemaRef, batch: Batch) -> Vec<(String, St ]); let evaluation = Operator::keyed_evaluation(build.schema(), 1, 8, output).unwrap(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input, vec![batch]).unwrap()) - .unwrap(); - dag.add(1, vec![0], build).unwrap(); - dag.add(2, vec![1], evaluation).unwrap(); + dag.add( + 0, + vec![], + Operator::source(input, vec![batch]).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add(1, vec![0], build, Lifetime::None).unwrap(); + dag.add(2, vec![1], evaluation, Lifetime::None).unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, @@ -290,10 +295,11 @@ fn int64_weight_column_equals_float64_column() { vec![batch(&int64, Some(Value::Int64((1 << 53) + 1)))], ) .unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], build).unwrap(); - dag.add(2, vec![1], evaluation).unwrap(); + dag.add(1, vec![0], build, Lifetime::None).unwrap(); + dag.add(2, vec![1], evaluation, Lifetime::None).unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, @@ -332,10 +338,16 @@ fn null_weight_adds_nothing() { 0, vec![], Operator::source(nullable.clone(), vec![nulls.clone()]).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add( + 1, + vec![0], + keyed_build(&nullable, RowWeight::Column(2)), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], keyed_build(&nullable, RowWeight::Column(2))) - .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 98c67cc59..c73b459c6 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -17,6 +17,7 @@ use planner_types::plan::physical_asap_dag::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; +use planner_types::stages::{Lifetime, StorageMedium}; use planner_types::summary_type::{ExactKind, ExactParams}; use std::sync::Arc; fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { @@ -72,23 +73,12 @@ fn with_table( } fn run(dag: &PhysicalDAG<'_, Batch, SchemaRef>, root: u64, scope: Scope) -> Vec> { - let context = RunContext::new( - scope, - Limits { - max_buffered_batches: 1, - ..Limits::default() - }, - ) - .unwrap(); - block_on(async { - let mut stream = dag.execute(&[root], context.clone()).unwrap().remove(0); - let mut rows = vec![]; - while let Some(batch) = stream.next().await { - rows.extend(batch.unwrap().to_rows().iter().cloned()); - } - assert_eq!(context.retained_bytes(), 0); - rows - }) + let context = RunContext::new(scope, Limits::default()).unwrap(); + let batches = crate::common::run_roots(dag, &[root], context.clone()) + .unwrap() + .remove(0); + assert_eq!(context.retained_bytes(), 0); + batches.iter().flat_map(Batch::to_rows).collect() } fn query() -> Scope { Scope::Query { @@ -134,6 +124,7 @@ fn grouped_sort_limit_across_batches() { 0, vec![], Operator::source(schema.clone(), batches).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -150,10 +141,16 @@ fn grouped_sort_limit_across_batches() { None, ) .unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add( + 2, + vec![1], + Operator::limit(schema, 1, 1, vec![0]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(2, vec![1], Operator::limit(schema, 1, 1, vec![0]).unwrap()) - .unwrap(); assert_eq!(floats(&run(&dag, 2, query()), 1), vec![5., 4.]); } @@ -169,15 +166,32 @@ fn summary_construction_merge_and_evaluation_at_both_phases() { Operator::per_group_summary_build(schema.clone(), family, None, 0, None, vec![]).unwrap(); let state = build.schema(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(schema, batches).unwrap()) - .unwrap(); - dag.add(1, vec![0], build).unwrap(); - dag.add(2, vec![1, 1], Operator::union(state.clone(), 2).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(schema, batches).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add( + 1, + vec![0], + build, + Lifetime::OneEvaluation(StorageMedium::Memory), + ) + .unwrap(); + dag.add( + 2, + vec![1, 1], + Operator::union(state.clone(), 2).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 3, vec![2], Operator::summary_merge(state.clone(), 0, vec![]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -195,6 +209,7 @@ fn summary_construction_merge_and_evaluation_at_both_phases() { ), ) .unwrap(), + Lifetime::None, ) .unwrap(); for scope in [ @@ -232,13 +247,15 @@ fn diamond_semijoin_preserves_left_values_and_multiplicity() { 0, vec![], Operator::source(schema.clone(), batches).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], filter).unwrap(); + dag.add(1, vec![0], filter, Lifetime::None).unwrap(); dag.add( 2, vec![0, 1], Operator::semi_join(schema.clone(), schema, vec![(0, 0)]).unwrap(), + Lifetime::None, ) .unwrap(); let rows = run(&dag, 2, query()); @@ -270,19 +287,26 @@ fn exact_integer_and_empty_extrema() { .unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], aggregate).unwrap(); + dag.add(1, vec![0], aggregate, Lifetime::None).unwrap(); assert!(matches!(run(&dag,1,query())[0][0],Value::Int64(v) if v==value+2)); let mut empty = PhysicalDAG::default(); empty - .add(0, vec![], Operator::source(schema.clone(), vec![]).unwrap()) + .add( + 0, + vec![], + Operator::source(schema.clone(), vec![]).unwrap(), + Lifetime::None, + ) .unwrap(); empty .add( 1, vec![0], Operator::aggregate(schema, vec![], vec![("min".into(), Reduction::Min(0))]).unwrap(), + Lifetime::None, ) .unwrap(); assert!(matches!(run(&empty, 1, query())[0][0], Value::Null)); @@ -302,9 +326,9 @@ fn scalar_negation_and_vector_conversion() { .unwrap(); let convert = Operator::vector_to_scalar(project.schema(), 0).unwrap(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], scalar).unwrap(); - dag.add(1, vec![0], project).unwrap(); - dag.add(2, vec![1], convert).unwrap(); + dag.add(0, vec![], scalar, Lifetime::None).unwrap(); + dag.add(1, vec![0], project, Lifetime::None).unwrap(); + dag.add(2, vec![1], convert, Lifetime::None).unwrap(); assert_eq!(floats(&run(&dag, 2, query()), 0), vec![-7.]); let scalar = Operator::scalar(Value::Float64(f64::NAN), DataType::Float64).unwrap(); let predicate = Expression::Equal( @@ -313,8 +337,8 @@ fn scalar_negation_and_vector_conversion() { ); let filter = Operator::filter(scalar.schema(), predicate).unwrap(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], scalar).unwrap(); - dag.add(1, vec![0], filter).unwrap(); + dag.add(0, vec![], scalar, Lifetime::None).unwrap(); + dag.add(1, vec![0], filter, Lifetime::None).unwrap(); assert!(run(&dag, 1, query()).is_empty()); } @@ -381,9 +405,10 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { 0, vec![], Operator::source(input.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], build.clone()).unwrap(); + dag.add(1, vec![0], build.clone(), Lifetime::None).unwrap(); run( &dag, 1, @@ -407,6 +432,7 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { vec![Batch::try_new(state.clone(), rows).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); states.push(0); @@ -426,21 +452,24 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { .unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(2, vec![1], build.clone()).unwrap(); + dag.add(2, vec![1], build.clone(), Lifetime::None).unwrap(); states.push(2); } dag.add( 3, states.clone(), Operator::union(state.clone(), states.len()).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 4, vec![3], Operator::summary_merge(state.clone(), 0, vec![]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -454,6 +483,7 @@ fn kll_raw_partial_and_precomputed_are_native_dags() { ), ) .unwrap(), + Lifetime::None, ) .unwrap(); floats(&run(&dag, 5, query()), 0)[0] @@ -496,6 +526,7 @@ fn exact_state_and_family_validation() { vec![Batch::try_new(schema.clone(), vec![vec![value]]).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -513,6 +544,7 @@ fn exact_state_and_family_validation() { ), ) .unwrap(), + Lifetime::None, ) .unwrap(); assert_eq!(floats(&run(&dag, 1, query()), 0), vec![7.]); @@ -640,10 +672,15 @@ fn empty_exact_count_is_an_integer_state_evaluation() { ) .unwrap(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input, vec![]).unwrap()) - .unwrap(); - dag.add(1, vec![0], build).unwrap(); - dag.add(2, vec![1], read).unwrap(); + dag.add( + 0, + vec![], + Operator::source(input, vec![]).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add(1, vec![0], build, Lifetime::None).unwrap(); + dag.add(2, vec![1], read, Lifetime::None).unwrap(); assert!(matches!(run(&dag, 2, query())[0][0], Value::Int64(0))); } @@ -664,6 +701,7 @@ fn extrema_preserve_numeric_values_in_the_presence_of_nan() { .unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -678,6 +716,7 @@ fn extrema_preserve_numeric_values_in_the_presence_of_nan() { ], ) .unwrap(), + Lifetime::None, ) .unwrap(); let rows = run(&dag, 1, query()); @@ -866,10 +905,11 @@ fn planner_expressions_preserve_collection_and_nullable_types() { .unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); let projected = project.schema(); - dag.add(1, vec![0], project).unwrap(); + dag.add(1, vec![0], project, Lifetime::None).unwrap(); let predicate = ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, left: Box::new(ScalarExpr::Column(0)), @@ -884,6 +924,7 @@ fn planner_expressions_preserve_collection_and_nullable_types() { Expression::planner(CompiledExpression::compile(&predicate, &projected).unwrap()), ) .unwrap(), + Lifetime::None, ) .unwrap(); let rows = run(&dag, 2, query()); @@ -953,6 +994,7 @@ fn native_relational_join_kinds_preserve_unmatched_rows() { vec![Batch::try_new(input.clone(), rows).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); } @@ -967,6 +1009,7 @@ fn native_relational_join_kinds_preserve_unmatched_rows() { output, ) .unwrap(), + Lifetime::None, ) .unwrap(); assert_eq!(run(&dag, 2, query()).len(), count, "{kind:?}"); @@ -1058,11 +1101,12 @@ fn assert_weighted_rate_topk(count_sketch: bool) { 0, vec![], Operator::source(raw.clone(), vec![Batch::try_new(raw, rows).unwrap()]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], rates).unwrap(); - dag.add(2, vec![1], build).unwrap(); - dag.add(3, vec![2], evaluation).unwrap(); + dag.add(1, vec![0], rates, Lifetime::None).unwrap(); + dag.add(2, vec![1], build, Lifetime::None).unwrap(); + dag.add(3, vec![2], evaluation, Lifetime::None).unwrap(); dag.add( 4, vec![3], @@ -1077,10 +1121,16 @@ fn assert_weighted_rate_topk(count_sketch: bool) { None, ) .unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add( + 5, + vec![4], + Operator::limit(output, 2, 0, vec![0]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(5, vec![4], Operator::limit(output, 2, 0, vec![0]).unwrap()) - .unwrap(); for scope in [ query(), Scope::Ingestion { @@ -1267,9 +1317,9 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { }; CompiledPhysicalDAG::from_operators( [ - (0, (vec![], source(left))), - (1, (vec![], source(right))), - (2, (vec![0, 1], join.clone())), + (0, (vec![], source(left), Lifetime::None)), + (1, (vec![], source(right), Lifetime::None)), + (2, (vec![0, 1], join.clone(), Lifetime::None)), ] .into(), vec![2], @@ -1389,7 +1439,14 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { }) .collect::>(); operators.insert(2, (vec![0, 1], binary.clone())); - let program = CompiledPhysicalDAG::from_operators(operators, vec![2]).unwrap(); + let program = CompiledPhysicalDAG::from_operators( + operators + .into_iter() + .map(|(id, (inputs, operator))| (id, (inputs, operator, Lifetime::None))) + .collect(), + vec![2], + ) + .unwrap(); let dag = program .instantiate(Arc::new(InMemoryDataStore::default())) .unwrap(); diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index c13d0ab70..5c7d25196 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -16,6 +16,7 @@ use planner_types::ir::NonASAPOp; use planner_types::ir::Predicate; use planner_types::ir::ScalarExpr; use planner_types::plan::physical_asap_dag::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; +use planner_types::stages::{Lifetime, StorageMedium}; use std::sync::Arc; fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { @@ -38,10 +39,7 @@ fn context() -> RunContext { Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_buffered_batches: 1, - ..Limits::default() - }, + Limits::default(), ) .unwrap() } @@ -64,9 +62,14 @@ fn unary(input: SchemaRef, batches: Vec>>, op: Operator) -> Vec]) -> Vec>> { @@ -108,10 +111,11 @@ fn join(left: Vec, right: Vec, kind: JoinKind, keyed: bool) -> Vec id, vec![], Operator::source(input.clone(), batches).unwrap(), + Lifetime::None, ) .unwrap(); } - dag.add(2, vec![0, 1], op).unwrap(); + dag.add(2, vec![0, 1], op, Lifetime::None).unwrap(); collect(&dag, 2) } @@ -844,16 +848,27 @@ fn limit_branch_finishes_without_blocking_shared_sibling() { let batches = (0..100) .map(|v| Batch::try_new(input.clone(), vec![vec![Value::Int64(v)]]).unwrap()) .collect(); - dag.add(0, vec![], Operator::source(input.clone(), batches).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), batches).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 1, vec![0], Operator::limit(input.clone(), 1, 0, vec![]).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add( + 2, + vec![0, 1], + Operator::union(input, 2).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(2, vec![0, 1], Operator::union(input, 2).unwrap()) - .unwrap(); // Bound polls as well as rows so a backpressure regression cannot hang the suite. use futures::{task::noop_waker_ref, Stream}; use std::{ @@ -977,6 +992,7 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { vec![Batch::try_new(input.clone(), rows).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -991,18 +1007,25 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { vec![], ) .unwrap(), + Lifetime::OneEvaluation(StorageMedium::Memory), ) .unwrap(); } let state = Operator::per_group_summary_build(input, family, Some(kll_512()), 0, None, vec![]) .unwrap() .schema(); - dag.add(6, vec![3, 4], Operator::union(state.clone(), 2).unwrap()) - .unwrap(); + dag.add( + 6, + vec![3, 4], + Operator::union(state.clone(), 2).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 7, vec![6], Operator::summary_merge(state.clone(), 0, vec![]).unwrap(), + Lifetime::OneEvaluation(StorageMedium::Memory), ) .unwrap(); let mut roots = vec![]; @@ -1020,6 +1043,7 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { ), ) .unwrap(), + Lifetime::None, ) .unwrap(); roots.push(id); @@ -1097,12 +1121,14 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { vec![Batch::try_new(input.clone(), rows).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); dag.add( TUMBLING_WINDOW_COUNT + tumbling_window, vec![tumbling_window], build(), + Lifetime::None, ) .unwrap(); } @@ -1111,12 +1137,14 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { union, (TUMBLING_WINDOW_COUNT..2 * TUMBLING_WINDOW_COUNT).collect(), Operator::union(state.clone(), TUMBLING_WINDOW_COUNT as usize).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( union + 1, vec![union], Operator::summary_merge(state.clone(), 0, vec![]).unwrap(), + Lifetime::None, ) .unwrap(); for (i, q) in [0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99] @@ -1135,6 +1163,7 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { }), ) .unwrap(), + Lifetime::None, ) .unwrap(); let Value::Float64(estimate) = collect(&dag, id)[0][0] else { @@ -1193,16 +1222,23 @@ fn counter_tumbling_windows_merge_in_any_order() { id, vec![], Operator::source(input.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(id + 4, vec![id], build()).unwrap(); + dag.add(id + 4, vec![id], build(), Lifetime::None).unwrap(); } - dag.add(8, vec![4, 5, 6], Operator::union(state.clone(), 3).unwrap()) - .unwrap(); + dag.add( + 8, + vec![4, 5, 6], + Operator::union(state.clone(), 3).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 9, vec![8], Operator::summary_merge(state, 0, vec![]).unwrap(), + Lifetime::None, ) .unwrap(); assert_eq!(increase(&dag, 9), increase(&dag, 7)); @@ -1215,16 +1251,18 @@ fn zero_column_output_obeys_memory_limit() { let input = schema(&[]); let batch = Batch::try_new(input.clone(), vec![vec![]; 200]).unwrap(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input, vec![batch]).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(input, vec![batch]).unwrap(), + Lifetime::None, + ) + .unwrap(); let run = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes: 1024, - ..Limits::default() - }, + Limits { max_bytes: 1024 }, ) .unwrap(); let mut stream = dag.execute(&[0], run.clone()).unwrap().remove(0); @@ -1257,9 +1295,14 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { .unwrap(); let state = build.schema(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input.clone(), vec![]).unwrap()) - .unwrap(); - dag.add(1, vec![0], build).unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), vec![]).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add(1, vec![0], build, Lifetime::None).unwrap(); dag.add( 2, vec![1], @@ -1275,6 +1318,7 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { ), ) .unwrap(), + Lifetime::None, ) .unwrap(); let rows = collect(&dag, 2); @@ -1491,10 +1535,11 @@ fn filtered_summary_build_keeps_empty_groups() { vec![Batch::try_new(input.clone(), rows.clone()).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], build).unwrap(); - dag.add(2, vec![1], read).unwrap(); + dag.add(1, vec![0], build, Lifetime::None).unwrap(); + dag.add(2, vec![1], read, Lifetime::None).unwrap(); assert_eq!(printed(&collect(&dag, 2)), expected); } } @@ -1711,9 +1756,10 @@ fn exact_frequency_rejects_non_finite_identities() { vec![Batch::try_new(input, vec![vec![Value::Float64(f64::INFINITY)]]).unwrap()], ) .unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], operator).unwrap(); + dag.add(1, vec![0], operator, Lifetime::None).unwrap(); let run = context(); let result = block_on(async { let mut stream = dag.execute(&[1], run.clone()).unwrap().remove(0); diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 7d908bb4e..d92c4f31e 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -5,7 +5,7 @@ use asap_executor::{ runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; -use futures::{executor::block_on, StreamExt}; + use planner_types::ir::operator::{GroupKeys, Reduction}; use planner_types::ir::properties::*; use planner_types::ir::scalar::ColumnRef; @@ -217,14 +217,8 @@ fn native_run( // run's end. store.append_materialized_view(view, (2000 - 60_000, 2000), vec![input])?; let physical_dag = program.instantiate(Arc::new(store))?; - block_on(async { - let mut rows = Vec::new(); - let mut stream = physical_dag.execute(program.roots(), context)?.remove(0); - while let Some(batch) = stream.next().await { - rows.extend(batch?.to_rows().iter().cloned()); - } - Ok(rows) - }) + let batches = crate::common::run_roots(&physical_dag, program.roots(), context)?.remove(0); + Ok(batches.iter().flat_map(Batch::to_rows).collect()) } fn ingestion_context(limits: Limits) -> RunContext { RunContext::new( @@ -356,10 +350,7 @@ fn precompute_dag_enforces_cancellation_and_budget() { &program, family, vec![sum_state(1.)], - ingestion_context(Limits { - max_bytes: 1, - ..Limits::default() - }), + ingestion_context(Limits { max_bytes: 1 }), ) .unwrap_err(); assert!(format!("{error:?}").contains("MemoryLimit")); diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 825f9f953..e0f6dd706 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -17,6 +17,7 @@ use planner_types::plan::physical_asap_dag::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; +use planner_types::stages::Lifetime; use std::{collections::BTreeMap, sync::Arc, time::Duration}; fn schema() -> SchemaRef { @@ -95,9 +96,9 @@ fn over_sources(binary: Operator, left: Batch, right: Batch) -> Arc) -> Result = crate::common::run_roots(&bound, &[2], ctx.clone()) + .unwrap() + .remove(0) + .iter() + .flat_map(Batch::to_rows) + .collect(); assert!( matches!(rows.as_slice(), [a,b] if matches!(a.as_slice(), [Value::Int64(9)]) && matches!(b.as_slice(), [Value::Int64(3)])) ); @@ -235,7 +233,8 @@ impl DataStore for CountingStore { no_data(view) } } -// Binding and cancellation do not perform I/O; fan-out opens one cursor per run. +// Binding and cancellation do not perform I/O; at query time each consumer +// of a raw data slot opens its own cursor. #[test] fn lazy_open_shared_producer_and_cancellation() { let (scan, schema, _) = fixture(); @@ -260,7 +259,7 @@ fn lazy_open_shared_producer_and_cancellation() { assert!(all.iter().flatten().all(Result::is_ok)); }); } - assert_eq!(opened.load(Ordering::SeqCst), 2); + assert_eq!(opened.load(Ordering::SeqCst), 4); } // Unsupported predicates fail before opening any cursor; a store without // the source and a failing reader fail the run, not as an empty result. @@ -340,10 +339,7 @@ fn schema_drift_and_memory_limits_fail_the_scan() { Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes: 1, - max_buffered_batches: 1, - }, + Limits { max_bytes: 1 }, ) .unwrap(); block_on(async { @@ -991,12 +987,7 @@ fn a_materialized_view_is_requested_for_the_window_it_summarizes() { Limits::default(), ) .unwrap(); - block_on(async { - let mut output = physical_dag.execute(&[4], ingestion).unwrap().remove(0); - while let Some(batch) = output.next().await { - batch.unwrap(); - } - }); + crate::common::run_roots(&physical_dag, &[4], ingestion).unwrap(); assert_eq!( store.time_ranges_ms.lock().unwrap()[1], (window_end_ms - 5 * MINUTE_MS, window_end_ms) diff --git a/crates/executor/tests/series_window_memory_limit.rs b/crates/executor/tests/series_window_memory_limit.rs index 07ca7d29f..31b58cfca 100644 --- a/crates/executor/tests/series_window_memory_limit.rs +++ b/crates/executor/tests/series_window_memory_limit.rs @@ -8,6 +8,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, Value}; use asap_executor::Error; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -76,16 +77,20 @@ fn a_long_subquery_over_many_series_fails_near_the_memory_limit() { ) .unwrap(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input, vec![batch]).unwrap()) - .unwrap(); - dag.add(1, vec![0], window).unwrap(); + dag.add( + 0, + vec![], + Operator::source(input, vec![batch]).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add(1, vec![0], window, Lifetime::None).unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 3_600_000, }, Limits { max_bytes: LIMIT_BYTES, - ..Limits::default() }, ) .unwrap(); diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index bcfd31660..71a3f4c7d 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -19,6 +19,7 @@ use asap_executor::summary_kernels::{ use asap_executor::values::{Batch, SchemaRef, Value}; use asap_executor::{AggregateCore, StateColumnUpdater}; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::collections::BTreeMap; use std::num::NonZeroU32; use std::sync::Arc; @@ -228,9 +229,10 @@ fn batch_build(build: Operator) -> BTreeMap { 0, vec![], Operator::source(input_schema(), vec![input_batch()]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(1, vec![0], build).unwrap(); + dag.add(1, vec![0], build, Lifetime::None).unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 10_000, diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 49886383b..ab3ea4b89 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -11,6 +11,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::{Error, StateColumnUpdater}; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::sync::Arc; const SERIES: u32 = 1_000; @@ -80,22 +81,21 @@ fn merge_stored_tumbling_windows(max_bytes: usize) -> Result { 0, vec![], Operator::source(schema.clone(), vec![tumbling_window; TUMBLING_WINDOW_COUNT]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 1, vec![0], Operator::summary_merge(schema, 1, vec![0]).unwrap(), + Lifetime::None, ) .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes, - ..Limits::default() - }, + Limits { max_bytes }, ) .unwrap(); block_on(async { @@ -141,22 +141,21 @@ fn compacting_a_stored_tumbling_window_does_not_charge_its_shared_sketches() { 0, vec![], Operator::source(schema.clone(), vec![tumbling_window]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 1, vec![0], Operator::limit(schema, kept, 0, vec![]).unwrap(), + Lifetime::None, ) .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes: 8 << 20, - ..Limits::default() - }, + Limits { max_bytes: 8 << 20 }, ) .unwrap(); let rows = block_on(async { @@ -184,6 +183,7 @@ fn sorting_then_limiting_stored_tumbling_windows_does_not_charge_their_sketches( 0, vec![], Operator::source(schema.clone(), vec![first, second]).unwrap(), + Lifetime::None, ) .unwrap(); let key = SortKey { @@ -195,18 +195,21 @@ fn sorting_then_limiting_stored_tumbling_windows_does_not_charge_their_sketches( 1, vec![0], Operator::sort(schema.clone(), vec![key], vec![], None).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add( + 2, + vec![1], + Operator::limit(schema, 980, 0, vec![]).unwrap(), + Lifetime::None, ) .unwrap(); - dag.add(2, vec![1], Operator::limit(schema, 980, 0, vec![]).unwrap()) - .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes: 8 << 20, - ..Limits::default() - }, + Limits { max_bytes: 8 << 20 }, ) .unwrap(); let rows = block_on(async { @@ -231,22 +234,21 @@ fn a_merge_passing_stored_states_through_does_not_charge_them() { 0, vec![], Operator::source(schema.clone(), vec![tumbling_window]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 1, vec![0], Operator::summary_merge(schema, 1, vec![0]).unwrap(), + Lifetime::None, ) .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes: 8 << 20, - ..Limits::default() - }, + Limits { max_bytes: 8 << 20 }, ) .unwrap(); let rows = block_on(async { @@ -410,7 +412,6 @@ fn a_materialized_view_providers_states_are_not_charged_to_the_run() { }, Limits { max_bytes: 32 << 20, - ..Limits::default() }, ) .unwrap(); diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index ea30af325..19d3b4ac5 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -9,6 +9,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, Value}; use asap_executor::Error; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -61,6 +62,7 @@ fn one_large_batch_fails_near_the_memory_limit() { 0, vec![], Operator::source(input.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -75,6 +77,7 @@ fn one_large_batch_fails_near_the_memory_limit() { vec![0], ) .unwrap(), + Lifetime::None, ) .unwrap(); let context = RunContext::new( @@ -83,7 +86,6 @@ fn one_large_batch_fails_near_the_memory_limit() { }, Limits { max_bytes: LIMIT_BYTES, - ..Limits::default() }, ) .unwrap(); diff --git a/crates/executor/tests/text_extremum_memory.rs b/crates/executor/tests/text_extremum_memory.rs index 0cc056fa4..221919d64 100644 --- a/crates/executor/tests/text_extremum_memory.rs +++ b/crates/executor/tests/text_extremum_memory.rs @@ -9,6 +9,7 @@ use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::Error; use futures::{executor::block_on, StreamExt}; +use planner_types::stages::Lifetime; use std::sync::Arc; /// Run `measures` over `batch`, grouped by `groups`, under `max_bytes`; @@ -25,22 +26,21 @@ fn aggregate( 0, vec![], Operator::source(input.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 1, vec![0], Operator::aggregate(input, groups, measures).unwrap(), + Lifetime::None, ) .unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 0, }, - Limits { - max_bytes, - ..Limits::default() - }, + Limits { max_bytes }, ) .unwrap(); block_on(async { diff --git a/crates/executor/tests/time_series_window.rs b/crates/executor/tests/time_series_window.rs index ef24d850c..4772139fb 100644 --- a/crates/executor/tests/time_series_window.rs +++ b/crates/executor/tests/time_series_window.rs @@ -6,7 +6,7 @@ use asap_executor::physical_planner::{compile, promql_rows}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; -use futures::{executor::block_on, StreamExt}; + use planner_types::ir::operator::{Source, TimeShift}; use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; use planner_types::ir::schema::{DataType, Field, Schema, PROMQL_SERIES_IDENTITY}; @@ -136,19 +136,17 @@ fn points_of(dag: &PhysicalASAPDAG, store: InMemoryDataStore, scope: Scope) -> V .instantiate(Arc::new(store)) .unwrap(); let context = RunContext::new(scope, Limits::default()).unwrap(); - block_on(async { - let mut output = bound.execute(&roots, context).unwrap().remove(0); - let mut points = Vec::new(); - while let Some(batch) = output.next().await { - for row in batch.unwrap().to_rows() { - match (&row[1], &row[2]) { - (Value::Timestamp(ts), Value::Float64(v)) => points.push((*ts, *v)), - other => panic!("{other:?}"), - } - } + let mut points = Vec::new(); + let batches = crate::common::run_roots(&bound, &roots, context) + .unwrap() + .remove(0); + for row in batches.iter().flat_map(Batch::to_rows) { + match (&row[1], &row[2]) { + (Value::Timestamp(ts), Value::Float64(v)) => points.push((*ts, *v)), + other => panic!("{other:?}"), } - points - }) + } + points } fn at(evaluation_time_ms: i64) -> Scope { diff --git a/crates/executor/tests/tumbling_window_chain.rs b/crates/executor/tests/tumbling_window_chain.rs index e6c3950de..9a9bee44d 100644 --- a/crates/executor/tests/tumbling_window_chain.rs +++ b/crates/executor/tests/tumbling_window_chain.rs @@ -14,6 +14,7 @@ use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use planner_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, TimeRangeKind}; use planner_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use planner_types::stages::Lifetime; use planner_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; use std::collections::BTreeMap; use std::rc::Rc; @@ -143,19 +144,11 @@ fn run( roots: &[u64], context: RunContext, ) -> Vec>> { - // The roots share producers, so they are read together. - block_on(futures::future::join_all( - dag.execute(roots, context) - .unwrap() - .into_iter() - .map(|mut output| async move { - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - rows.extend(batch.unwrap().to_rows()); - } - rows - }), - )) + crate::common::run_roots(dag, roots, context) + .unwrap() + .into_iter() + .map(|batches| batches.iter().flat_map(Batch::to_rows).collect()) + .collect() } /// The id of the node of `dag` that is `operator`'s node in `nodes`. @@ -265,13 +258,7 @@ fn a_chain_built_at_ingestion_answers_queries_through_its_time_ranges() { window_end += WIDTH_MS; } let evaluation_time_ms = 4 * HOUR_MS; - let context = RunContext::new( - Scope::Query { evaluation_time_ms }, - Limits { - ..Limits::default() - }, - ) - .unwrap(); + let context = RunContext::new(Scope::Query { evaluation_time_ms }, Limits::default()).unwrap(); let answers = { let query = query.instantiate(store.clone()).unwrap(); run(&query, &[now as u64, shifted as u64], context) @@ -405,13 +392,14 @@ fn a_range_window_stamps_each_row_with_its_tumbling_windows_end() { 0, vec![], ExecutorOperator::source(schema.clone(), vec![batch]).unwrap(), + Lifetime::None, ) .unwrap(); let window = ExecutorOperator::time_range(schema, 1, 0, 3 * MINUTE_MS, None, None) .unwrap() .with_tumbling_window_ends(MINUTE_MS) .unwrap(); - dag.add(1, vec![0], window).unwrap(); + dag.add(1, vec![0], window, Lifetime::None).unwrap(); let context = RunContext::new( Scope::Query { evaluation_time_ms: 10 * MINUTE_MS, diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index c6fa4aff3..1866ac810 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -16,6 +16,7 @@ use planner_types::ir::schema::{DataType, Schema}; use planner_types::ir::schema::{Field, FieldDataType}; use planner_types::ir::ASAPOp; use planner_types::plan::physical_asap_dag::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; +use planner_types::stages::{Lifetime, StorageMedium}; use planner_types::summary_type::{ SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, }; @@ -60,8 +61,14 @@ fn three_statistics(batch: Batch, build: Operator) -> Result, Error> { 0, vec![], Operator::source(batch.schema().clone(), vec![batch])?, + Lifetime::None, + )?; + dag.add( + 1, + vec![0], + build, + Lifetime::OneEvaluation(StorageMedium::Memory), )?; - dag.add(1, vec![0], build)?; for (id, statistic) in [ (2, SketchStatistic::Cardinality), (3, SketchStatistic::FrequencyL2), @@ -71,6 +78,7 @@ fn three_statistics(batch: Batch, build: Operator) -> Result, Error> { id, vec![1], Operator::evaluation(state.clone(), 0, SummaryEvaluation::Sketch(statistic))?, + Lifetime::None, )?; } let outputs = block_on(futures::future::join_all( diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index 714b69c8c..f4a27a1db 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -18,6 +18,7 @@ use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::plan::materialized_view::MaterializedView; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::plan::physical_asap_dag::ViewCoverage; +use asap_types::stages::{Lifetime, StorageMedium}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; use futures::{executor::block_on, StreamExt}; use std::{ @@ -346,6 +347,7 @@ fn five_tumbling_windows_roundtrip_and_shared_merge_runs_once() { 0, vec![], Operator::source(schema.clone(), vec![restored]).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( @@ -355,12 +357,23 @@ fn five_tumbling_windows_roundtrip_and_shared_merge_runs_once() { operator: merge.clone(), starts: starts.clone(), }, + Lifetime::OneEvaluation(StorageMedium::Memory), + ) + .unwrap(); + dag.add( + 2, + vec![1], + evaluation(merge.output_schema(), 0.5), + Lifetime::None, + ) + .unwrap(); + dag.add( + 3, + vec![1], + evaluation(merge.output_schema(), 0.99), + Lifetime::None, ) .unwrap(); - dag.add(2, vec![1], evaluation(merge.output_schema(), 0.5)) - .unwrap(); - dag.add(3, vec![1], evaluation(merge.output_schema(), 0.99)) - .unwrap(); let outputs = block_on(futures::future::join_all( dag.execute( &[2, 3], diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 0f8b39835..db20bb72f 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -4,7 +4,7 @@ use asap_executor::{ sources::InMemoryDataStore, values::Batch, }; -use futures::{executor::block_on, StreamExt}; +use futures::{executor::block_on, StreamExt, TryStreamExt}; use std::sync::Arc; /// Run `plan` against `store` and collect each root's batches. @@ -15,22 +15,22 @@ pub fn execute( scope: Scope, ) -> Vec> { let dag = plan.instantiate(Arc::new(store)).unwrap(); - block_on(async { - let streams = dag - .execute( - plan.roots(), - RunContext::new(scope, Limits::default()).unwrap(), - ) - .unwrap(); - futures::future::join_all(streams.into_iter().map(|mut stream| async move { - let mut batches = Vec::new(); - while let Some(batch) = stream.next().await { - batches.push((*batch.unwrap()).clone()); - } - batches - })) - .await - }) + let context = RunContext::new(scope, Limits::default()).unwrap(); + let outputs = match context.scope { + Scope::Ingestion { .. } => { + block_on(dag.execute_ingestion(plan.roots(), context).unwrap()).unwrap() + } + Scope::Query { .. } => dag + .execute(plan.roots(), context) + .unwrap() + .into_iter() + .map(|stream| block_on(stream.try_collect::>()).unwrap()) + .collect(), + }; + outputs + .into_iter() + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .collect() } /// Execute a relational DAG at query time against an in-memory connector diff --git a/crates/planner/tests/columnar_batches.rs b/crates/planner/tests/columnar_batches.rs index 2598a088f..6451a6272 100644 --- a/crates/planner/tests/columnar_batches.rs +++ b/crates/planner/tests/columnar_batches.rs @@ -21,7 +21,6 @@ use asap_planner::{ plan_installable, DeploymentCapabilities, FrontendInput, PlanningModels, PlanningWorkload, UserInput, }; -use futures::StreamExt; use crate::time_series; use time_series::{series_identity, RawDataSample}; @@ -163,13 +162,12 @@ async fn a_tumbling_window_is_built_from_arrow_columns_with_one_label_entry_per_ .unwrap(); // The built tumbling window: one row per series, its KLL in a state column. let tumbling_window = build - .execute(compiled.roots(), context) + .execute_ingestion(compiled.roots(), context) .unwrap() - .remove(0) - .next() .await .unwrap() - .unwrap(); + .remove(0) + .remove(0); let state_column = tumbling_window .schema() .fields diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 77d9656cd..309a83ace 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -23,7 +23,7 @@ use asap_types::ir::operator::Source; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ASAPOp, NonASAPOp}; use asap_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; -use futures::{executor::block_on, StreamExt}; +use futures::{executor::block_on, TryStreamExt}; const MINUTE_MS: i64 = 60_000; /// The evaluation time of every run. @@ -205,18 +205,25 @@ fn run_bound( } } let instance = compiled.instantiate(Arc::new(store)).unwrap(); - block_on(async { - let context = RunContext::new(scope, Limits::default()).unwrap(); - let outputs = instance.execute(compiled.roots(), context).unwrap(); - futures::future::join_all(outputs.into_iter().map(|mut output| async move { - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - batches.push((*batch.unwrap()).clone()); - } - batches - })) - .await - }) + let context = RunContext::new(scope, Limits::default()).unwrap(); + let outputs = match context.scope { + Scope::Ingestion { .. } => block_on( + instance + .execute_ingestion(compiled.roots(), context) + .unwrap(), + ) + .unwrap(), + Scope::Query { .. } => instance + .execute(compiled.roots(), context) + .unwrap() + .into_iter() + .map(|output| block_on(output.try_collect::>()).unwrap()) + .collect(), + }; + outputs + .into_iter() + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .collect() } /// The data plane of `plan` at [`EVALUATION_MS`] over the table rows diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index afc2ae94b..e96d17d98 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -673,10 +673,12 @@ impl PhysicalASAPDAG { } } -/// Test-only: `roots` exported as one DAG with every node at query time and -/// not materialized, and no θ (every `parameters` is `None`), with each -/// exported node's operator node by node id. Executor tests hand-build DAGs -/// this way; planning builds the plan in Stage 3. +/// Test-only: `roots` exported as one DAG with every node at query time, and +/// no θ (every `parameters` is `None`), with each exported node's operator +/// node by node id. A computed node with more than one consumer (edges and +/// root positions) is held for one evaluation, as Stage 3 selects for a +/// shared query-time output; every other node is not materialized. Executor tests hand-build +/// DAGs this way; planning builds the plan in Stage 3. #[cfg(any(test, feature = "test-support"))] pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( roots: &[crate::ir::QueryRoot], @@ -716,10 +718,26 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( view_coverage: None, }) .collect(); - Ok(( - PhysicalASAPDAG::new(nodes, roots, scalar_roots)?, - nodes_by_id, - )) + let mut dag = PhysicalASAPDAG::new(nodes, roots, scalar_roots)?; + let mut consumers = vec![0usize; dag.nodes.len()]; + let mut computed = vec![false; dag.nodes.len()]; + for edge in &dag.edges { + computed[edge.consumer] = true; + } + for producer in dag + .edges + .iter() + .map(|edge| edge.producer) + .chain(dag.roots.iter().copied()) + { + consumers[producer] += 1; + } + for node in &mut dag.nodes { + if consumers[node.id] > 1 && computed[node.id] { + node.lifetime = Lifetime::OneEvaluation(crate::stages::StorageMedium::Memory); + } + } + Ok((dag, nodes_by_id)) } fn grouping_compatibility( From 3697ec680cafe0eb9d5a9f93b4c31b4c55d2a0c7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:53:50 +0000 Subject: [PATCH 2/7] refactor(executor): one run rule per node, holding at query time only Address the quality review of the Lifetime runtime: - `run_as` returns `NodeRun { ReadPerConsumer, Held, Streamed }`, the one rule for per-consumer leaf reads, held outputs and the query-time fan-out check; holding now happens at query time only, so ingestion streams. - `start` walks the DAG once (`reach`): validates each node's run context, counts its consumers and records its inputs' context per evaluation time. - Error messages name the call for a wrong scope, and the fan-out error gives the evaluation time and says the node should be OneEvaluation. - The LIMIT test runs at ingestion and checks that the LIMIT consumer of a fan-out node drops its read; scope rejections get their own test. - Docs on `add`, `add_boxed`, `execute`, `execute_ingestion` and the README state the rejections; test helper and doc nits. --- crates/executor/README.md | 13 +- crates/executor/src/plan/mod.rs | 10 +- crates/executor/src/runtime/mod.rs | 246 +++++++++++--------- crates/executor/src/runtime/tests.rs | 76 +++--- crates/executor/tests/physical_semantics.rs | 53 ++--- crates/types/src/plan/physical_asap_dag.rs | 10 +- 6 files changed, 227 insertions(+), 181 deletions(-) diff --git a/crates/executor/README.md b/crates/executor/README.md index c76670a39..ed7f1140f 100644 --- a/crates/executor/README.md +++ b/crates/executor/README.md @@ -25,13 +25,16 @@ nodes below it at another time), as its `Lifetime` and the run's scope say: - Query time, `None`: streamed to its one consumer. A computed node with more than one consumer is a planner bug; `execute` returns an error naming the node before any operator starts. -- Ingestion time, a node with several consumers (one per-row node feeding - several chain builds or views): the node runs once and each batch reaches - every consumer. `execute_ingestion` polls every root itself and returns one - future that resolves to each root's batches, for the caller to write. +- Ingestion time: every node streams, whatever its lifetime. A node with + several consumers (one per-row node feeding several chain builds or views) + runs once and each batch reaches every consumer. `execute_ingestion` polls + every root itself and returns one future that resolves to each root's + batches, for the caller to write. `execute` runs a query and returns one stream per root; the caller may drain -them in any order. Dropping one consumer does not cancel other consumers. +them in any order. It rejects an ingestion scope, which `execute_ingestion` +runs, and `execute_ingestion` rejects a query scope. Dropping one consumer +does not cancel other consumers. `operators::Operator` implements native batch sources, scalar values, projection, filtering, grouped exact aggregation, semi-join, grouped Sort and diff --git a/crates/executor/src/plan/mod.rs b/crates/executor/src/plan/mod.rs index 4cbeb084b..0f868074d 100644 --- a/crates/executor/src/plan/mod.rs +++ b/crates/executor/src/plan/mod.rs @@ -63,6 +63,7 @@ impl Default for PhysicalDAG<'_, V, S> { } } impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { + /// Add node `id`, reading `inputs`, run by its plan `lifetime`. pub fn add( &mut self, id: NodeId, @@ -72,6 +73,7 @@ impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { ) -> Result<(), Error> { self.add_boxed(id, inputs, Box::new(operator), lifetime) } + /// [`Self::add`] with an already boxed operator. pub fn add_boxed( &mut self, id: NodeId, @@ -167,7 +169,10 @@ impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { .collect()) } /// A query run: one output stream per root, which the caller may drain - /// in any order. + /// in any order. Rejects an ingestion scope (run it with + /// [`Self::execute_ingestion`]), and a computed node with + /// `Lifetime::None` and more than one consumer at one evaluation time (a + /// planner bug), before any operator starts. pub fn execute<'r>( &'r self, roots: &[NodeId], @@ -179,7 +184,8 @@ impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { crate::runtime::execute(self, roots, context) } /// An ingestion run: one future that resolves to every root's batches, - /// in `roots` order, for the caller to write. + /// in `roots` order, for the caller to write. Rejects a query scope (run + /// it with [`Self::execute`]). pub fn execute_ingestion<'r>( &'r self, roots: &[NodeId], diff --git a/crates/executor/src/runtime/mod.rs b/crates/executor/src/runtime/mod.rs index 8f8b08f5e..5bac8e1b5 100644 --- a/crates/executor/src/runtime/mod.rs +++ b/crates/executor/src/runtime/mod.rs @@ -44,9 +44,9 @@ impl SharedValue { } } -/// A query-time run: one output stream per root. Fails before any operator -/// starts when a computed node that is not materialized -/// ([`Lifetime::None`]) has more than one consumer. +/// A query-time run: one output stream per root. Rejects an ingestion +/// scope, and a computed node with [`Lifetime::None`] and more than one +/// consumer (a planner bug), before any operator starts. pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( dag: &'r PhysicalDAG<'_, V, S>, roots: &[NodeId], @@ -54,7 +54,7 @@ pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( ) -> Result>, Error> { if context.evaluation_time_ms().is_none() { return Err(Error::Invalid( - "an ingestion run is executed by `execute_ingestion`".into(), + "`execute` takes a query scope; run an ingestion scope with `execute_ingestion`".into(), )); } start(dag, roots, context) @@ -62,7 +62,8 @@ pub(crate) fn execute<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( /// An ingestion run: every root's batches, the outputs the caller writes. /// The runtime polls the roots together, so a node feeding several of them -/// is computed once and each batch reaches every consumer. +/// is computed once and each batch reaches every consumer. Rejects a query +/// scope. pub(crate) fn execute_ingestion<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( dag: &'r PhysicalDAG<'_, V, S>, roots: &[NodeId], @@ -70,7 +71,7 @@ pub(crate) fn execute_ingestion<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( ) -> Result> + 'r, Error> { if context.evaluation_time_ms().is_some() { return Err(Error::Invalid( - "a query run is executed by `execute`".into(), + "`execute_ingestion` takes an ingestion scope; run a query scope with `execute`".into(), )); } let outputs = start(dag, roots, context)?; @@ -83,11 +84,118 @@ pub(crate) fn execute_ingestion<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( /// evaluates it at another time than the run's. type ProducerKey = (NodeId, Option); -/// Whether each consumer of `node` reads it on its own: at query time, a -/// leaf (a store read, a raw data or materialized view slot) is read once -/// per consumer, as the planner charges a not-materialized read per edge. -fn read_per_consumer(node: &Node<'_, V, S>, context: &RunContext) -> bool { - node.inputs.is_empty() && context.evaluation_time_ms().is_some() +/// How a node runs in a run, from the run's scope, whether it is a leaf and +/// its plan `Lifetime` (see the crate README). +#[derive(Clone, Copy, PartialEq, Eq)] +enum NodeRun { + /// Query time, a leaf (a raw data or materialized view slot): each + /// consumer reads the store on its own, as the planner charges a + /// not-materialized read on each edge. + ReadPerConsumer, + /// Query time, a materialized computed node: run to the end before any + /// consumer reads it, then read by each consumer. + Held, + /// Every other node: one run whose batches reach each consumer as they + /// come. At query time a computed one has a single consumer. + Streamed, +} +fn run_as(node: &Node<'_, V, S>, context: &RunContext) -> NodeRun { + let query = context.evaluation_time_ms().is_some(); + match (query, node.inputs.is_empty(), node.lifetime) { + (true, true, _) => NodeRun::ReadPerConsumer, + (true, false, Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_)) => NodeRun::Held, + _ => NodeRun::Streamed, + } +} + +/// Each (node, evaluation time) a run reaches: its consumer count and the +/// context its inputs run in. +type Reached = BTreeMap; + +/// Visit `id` once per evaluation time, before any operator starts: +/// validate its run context, count its consumers and record its inputs' +/// context. +fn reach( + dag: &PhysicalDAG<'_, V, S>, + id: NodeId, + context: &RunContext, + reached: &mut Reached, +) -> Result<(), Error> { + let key = (id, context.evaluation_time_ms()); + if let Some((count, _)) = reached.get_mut(&key) { + *count += 1; + return Ok(()); + } + let node = &dag.nodes[&id]; + node.operator.validate_context(context)?; + let input_context = node + .operator + .input_context(context) + .map_err(|source| Error::AtNode { + node: id, + operation: node.operator.name().into(), + source: Box::new(source), + })?; + for &child in &node.inputs { + reach(dag, child, &input_context, reached)?; + } + reached.insert(key, (1, input_context)); + Ok(()) +} + +fn build<'r, V: 'r, S: 'r>( + dag: &'r PhysicalDAG<'_, V, S>, + id: NodeId, + context: &RunContext, + reached: &Reached, + states: &mut BTreeMap>>>, +) -> Result>>, Error> { + let key = (id, context.evaluation_time_ms()); + let node = &dag.nodes[&id]; + let run = run_as(node, context); + if run != NodeRun::ReadPerConsumer { + if let Some(state) = states.get(&key) { + return Ok(Rc::clone(state)); + } + } + let input_context = &reached[&key].1; + let mut inputs = Vec::new(); + for &child in &node.inputs { + inputs.push(Input::subscribe(build( + dag, + child, + input_context, + reached, + states, + )?)); + } + let stream = node + .operator + .start(inputs, context.clone()) + .map_err(|source| Error::AtNode { + node: id, + operation: node.operator.name().into(), + source: Box::new(source), + })?; + let op = node.operator.as_ref(); + let state = Rc::new(RefCell::new(Producer { + stream: Some(stream), + node: id, + operation: node.operator.name().into(), + size: Box::new(move |value| op.output_bytes(value)), + context: context.clone(), + hold_until_finished: run == NodeRun::Held, + unread: BTreeMap::new(), + next_reader: 0, + batches_polled: 0, + waiters: BTreeMap::new(), + finished: false, + failure: None, + })); + if run != NodeRun::ReadPerConsumer { + states.insert(key, Rc::clone(&state)); + } + Ok(state) } fn start<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( @@ -99,113 +207,24 @@ fn start<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( return Err(Error::Cancelled); } dag.validate(roots)?; - let mut pending = roots.to_vec(); - let mut visited = std::collections::BTreeSet::new(); - while let Some(id) = pending.pop() { - if visited.insert(id) { - let node = &dag.nodes[&id]; - node.operator.validate_context(&context)?; - pending.extend(node.inputs.iter().copied()); - } - } - fn input_context( - dag: &PhysicalDAG<'_, V, S>, - id: NodeId, - context: &RunContext, - ) -> Result { - let node = &dag.nodes[&id]; - node.operator - .input_context(context) - .map_err(|source| Error::AtNode { - node: id, - operation: node.operator.name().into(), - source: Box::new(source), - }) - } - // The consumers of each shared producer, counted before any operator - // starts. - fn count_consumers( - dag: &PhysicalDAG<'_, V, S>, - id: NodeId, - context: &RunContext, - consumers: &mut BTreeMap, - ) -> Result<(), Error> { - let count = consumers - .entry((id, context.evaluation_time_ms())) - .or_default(); - *count += 1; - if *count == 1 { - let input_context = input_context(dag, id, context)?; - for &child in &dag.nodes[&id].inputs { - count_consumers(dag, child, &input_context, consumers)?; - } - } - Ok(()) - } - let mut consumers = BTreeMap::new(); + let mut reached = Reached::new(); for &root in roots { - count_consumers(dag, root, &context, &mut consumers)?; + reach(dag, root, &context, &mut reached)?; } - if context.evaluation_time_ms().is_some() { - for (&(id, _), &count) in &consumers { - let node = &dag.nodes[&id]; - if count > 1 && !node.inputs.is_empty() && node.lifetime == Lifetime::None { - return Err(Error::Invalid(format!( - "node {id} ({}) is not materialized but has {count} consumers at query time", - node.operator.name() - ))); - } - } - } - fn build<'r, V: 'r, S: 'r>( - dag: &'r PhysicalDAG<'_, V, S>, - id: NodeId, - context: &RunContext, - states: &mut BTreeMap>>>, - ) -> Result>>, Error> { - let key = (id, context.evaluation_time_ms()); + for (&(id, time), (count, _)) in &reached { let node = &dag.nodes[&id]; - let shared = !read_per_consumer(node, context); - if let Some(state) = states.get(&key).filter(|_| shared) { - return Ok(Rc::clone(state)); - } - let input_context = input_context(dag, id, context)?; - let mut inputs = Vec::new(); - for &child in &node.inputs { - inputs.push(Input::subscribe(build(dag, child, &input_context, states)?)); - } - let stream = node - .operator - .start(inputs, context.clone()) - .map_err(|source| Error::AtNode { - node: id, - operation: node.operator.name().into(), - source: Box::new(source), - })?; - let op = node.operator.as_ref(); - let state = Rc::new(RefCell::new(Producer { - stream: Some(stream), - node: id, - operation: node.operator.name().into(), - size: Box::new(move |value| op.output_bytes(value)), - context: context.clone(), - hold_until_finished: node.lifetime != Lifetime::None && !node.inputs.is_empty(), - unread: BTreeMap::new(), - next_reader: 0, - batches_polled: 0, - waiters: BTreeMap::new(), - finished: false, - failure: None, - })); - if shared { - states.insert(key, Rc::clone(&state)); + if let (Some(time), NodeRun::Streamed, 2..) = (time, run_as(node, &context), *count) { + return Err(Error::Invalid(format!( + "node {id} ({}) at evaluation time {time} has {count} consumers but \ + Lifetime::None; a shared query-time output must be OneEvaluation (planner bug)", + node.operator.name() + ))); } - Ok(state) } let mut states = BTreeMap::new(); roots .iter() - .map(|&id| build(dag, id, &context, &mut states).map(Input::subscribe)) + .map(|&id| build(dag, id, &context, &reached, &mut states).map(Input::subscribe)) .collect() } /// One run of a node, read by every consumer it has in the run. @@ -215,10 +234,9 @@ struct Producer<'a, V> { stream: Option>, size: Box usize + 'a>, context: RunContext, - /// A materialized node ([`Lifetime::OneEvaluation`] or - /// [`Lifetime::AcrossEvaluations`]) is computed to the end before any - /// consumer reads it, and its batches stay charged to the run until - /// every consumer has read them. + /// At query time, a materialized node ([`NodeRun::Held`]) is computed + /// to the end before any consumer reads it, and its batches stay + /// charged to the run until every consumer has read them. hold_until_finished: bool, /// The batches each reader has not read yet, by reader. unread: BTreeMap>>, diff --git a/crates/executor/src/runtime/tests.rs b/crates/executor/src/runtime/tests.rs index a10864712..c9e2fd9a1 100644 --- a/crates/executor/src/runtime/tests.rs +++ b/crates/executor/src/runtime/tests.rs @@ -62,12 +62,13 @@ impl PhysicalOperator for Identity { .boxed_local()) } } -fn context() -> RunContext { +/// A query run at time 100 with a byte budget of `max_bytes`. +fn query_context(max_bytes: usize) -> RunContext { RunContext::new( Scope::Query { evaluation_time_ms: 100, }, - Limits { max_bytes: 1024 }, + Limits { max_bytes }, ) .unwrap() } @@ -95,7 +96,7 @@ fn query_time_leaves_are_read_per_consumer() { let (source, starts, polls) = source(false); let mut dag = PhysicalDAG::default(); dag.add(0, vec![], source, Lifetime::None).unwrap(); - let context = context(); + let context = query_context(1024); let mut readers = dag.execute(&[0, 0], context.clone()).unwrap(); let second = readers.pop().unwrap(); let mut first = readers.pop().unwrap(); @@ -122,7 +123,7 @@ fn materialized_node_runs_once_and_is_held() { dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); dag.add(3, vec![1], Identity, Lifetime::None).unwrap(); for _ in 0..2 { - let run = context(); + let run = query_context(1024); let outputs = dag.execute(&[2, 3], run.clone()).unwrap(); for output in outputs { assert_eq!( @@ -146,7 +147,7 @@ fn broadcast_error_and_cancel() { let mut dag = PhysicalDAG::default(); dag.add(0, vec![], source, Lifetime::None).unwrap(); dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); - let mut outputs = dag.execute(&[1, 1], context()).unwrap(); + let mut outputs = dag.execute(&[1, 1], query_context(1024)).unwrap(); let a = outputs.pop().unwrap(); let b = outputs.pop().unwrap(); let (a, b) = block_on(async { futures::join!(a.collect::>(), b.collect::>()) }); @@ -155,7 +156,7 @@ fn broadcast_error_and_cancel() { assert!(matches!(values[1], Err(Error::AtNode { node: 0, .. }))); } assert_eq!(polls.get(), 2); - let run = context(); + let run = query_context(1024); let mut output = dag.execute(&[0], run.clone()).unwrap().remove(0); run.cancel(); assert!(matches!( @@ -172,7 +173,7 @@ fn retained_outputs_count_against_budget() { let (source, _, _) = source(false); let mut dag = PhysicalDAG::default(); dag.add(0, vec![], source, Lifetime::None).unwrap(); - let run = limits_context(8); + let run = query_context(8); let mut input = dag.execute(&[0], run.clone()).unwrap().remove(0); let held = block_on(input.next()).unwrap().unwrap(); assert_eq!(run.retained_bytes(), 8); @@ -194,7 +195,7 @@ fn invalid_dags_do_not_start_sources() { dag.add(0, vec![], source, Lifetime::None).unwrap(); dag.add(1, vec![2], Identity, Lifetime::None).unwrap(); dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); - assert!(dag.execute(&[0, 1], context()).is_err()); + assert!(dag.execute(&[0, 1], query_context(1024)).is_err()); assert_eq!(starts.get(), 0); let mut missing = PhysicalDAG::default(); missing.add(1, vec![9], Identity, Lifetime::None).unwrap(); @@ -211,7 +212,7 @@ fn ready_sources_cooperate_with_cancellation() { source.end = 10_000; let mut dag = PhysicalDAG::default(); dag.add(0, vec![], source, Lifetime::None).unwrap(); - let context = context(); + let context = query_context(1024); let mut input = dag.execute(&[0], context.clone()).unwrap().remove(0); block_on(async { let drain = async { @@ -319,15 +320,6 @@ fn two_readers_of_one_leaf(dag: &mut PhysicalDAG<'static, u64, ()>) -> Rc RunContext { - RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - Limits { max_bytes }, - ) - .unwrap() -} fn ingestion_context(max_bytes: usize) -> RunContext { RunContext::new( Scope::Ingestion { @@ -352,7 +344,7 @@ fn roots_reading_one_leaf_drain_in_order() { within_timeout(|| { let mut dag = PhysicalDAG::default(); two_readers_of_one_leaf(&mut dag); - let outputs = dag.execute(&[1, 2], limits_context(1024)).unwrap(); + let outputs = dag.execute(&[1, 2], query_context(1024)).unwrap(); let joined: Vec<_> = block_on(futures::future::join_all( outputs.into_iter().map(|output| output.collect::>()), )) @@ -360,7 +352,7 @@ fn roots_reading_one_leaf_drain_in_order() { .map(values) .collect(); assert_eq!(joined, vec![all_batches(), all_batches()]); - let run = limits_context(1024); + let run = query_context(1024); let outputs = dag.execute(&[1, 2], run.clone()).unwrap(); let drained: Vec<_> = outputs .into_iter() @@ -380,7 +372,7 @@ fn one_root_with_two_branches_over_one_leaf() { two_readers_of_one_leaf(&mut dag); dag.add(3, vec![1, 2], LeftThenRight, Lifetime::None) .unwrap(); - let output = dag.execute(&[3], limits_context(1024)).unwrap().remove(0); + let output = dag.execute(&[3], query_context(1024)).unwrap().remove(0); let both: Vec<_> = all_batches().into_iter().chain(all_batches()).collect(); assert_eq!(values(block_on(output.collect::>())), both); }); @@ -398,20 +390,42 @@ fn ingestion_fan_out_reads_the_source_once() { .unwrap(); let run = ingestion_context(1024); let outputs = block_on(dag.execute_ingestion(&[3, 4], run.clone()).unwrap()).unwrap(); - let outputs: Vec<_> = outputs + let outputs: Vec>> = outputs .into_iter() - .map(|batches| batches.into_iter().map(Ok).collect::>()) - .map(values) + .map(|batches| batches.into_iter().map(|v| Ok(*v)).collect()) .collect(); let both: Vec<_> = all_batches().into_iter().chain(all_batches()).collect(); assert_eq!(outputs, vec![all_batches(), both]); assert_eq!(starts.get(), 1); assert_eq!(run.retained_bytes(), 0); - assert!(dag.execute(&[3], ingestion_context(1024)).is_err()); - assert!(dag.execute_ingestion(&[3], limits_context(1024)).is_err()); }); } +// Each entry point runs one scope and names the other for the other scope. +#[test] +fn execute_rejects_ingestion_scope_and_vice_versa() { + let mut dag = PhysicalDAG::default(); + two_readers_of_one_leaf(&mut dag); + let Err(error) = dag.execute(&[1], ingestion_context(1024)) else { + panic!("`execute` ran an ingestion scope"); + }; + assert_eq!( + error, + Error::Invalid( + "`execute` takes a query scope; run an ingestion scope with `execute_ingestion`".into() + ) + ); + let Err(error) = dag.execute_ingestion(&[1], query_context(1024)) else { + panic!("`execute_ingestion` ran a query scope"); + }; + assert_eq!( + error, + Error::Invalid( + "`execute_ingestion` takes an ingestion scope; run a query scope with `execute`".into() + ) + ); +} + // A computed node that is not materialized has exactly one consumer at // query time; two mean a planner bug, reported before any operator starts. #[test] @@ -422,19 +436,21 @@ fn query_time_fan_out_of_a_not_materialized_node_is_an_error() { dag.add(1, vec![0], Identity, Lifetime::None).unwrap(); dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); dag.add(3, vec![1], Identity, Lifetime::None).unwrap(); - let Err(error) = dag.execute(&[2, 3], context()) else { + let Err(error) = dag.execute(&[2, 3], query_context(1024)) else { panic!("a not-materialized node with two consumers ran"); }; assert_eq!( error, Error::Invalid( - "node 1 (Identity) is not materialized but has 2 consumers at query time".into() + "node 1 (Identity) at evaluation time 100 has 2 consumers but Lifetime::None; \ + a shared query-time output must be OneEvaluation (planner bug)" + .into() ) ); assert_eq!(starts.get(), 0); // Read at two evaluation times, it runs once per time, each with one consumer. dag.add(4, vec![1], MinuteEarlier, Lifetime::None).unwrap(); - let outputs = dag.execute(&[2, 4], context()).unwrap(); + let outputs = dag.execute(&[2, 4], query_context(1024)).unwrap(); for output in outputs { assert_eq!(values(block_on(output.collect::>())).len(), 4); } @@ -450,7 +466,7 @@ fn one_evaluation_output_over_budget_is_a_memory_limit() { source.end = MANY_BATCHES; dag.add(0, vec![], source, Lifetime::None).unwrap(); dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); - let run = limits_context(5 * 8); + let run = query_context(5 * 8); let output = dag.execute(&[1], run.clone()).unwrap().remove(0); let read = values(block_on(output.collect::>())); assert_eq!(read.last(), Some(&Err(Error::MemoryLimit))); diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 5c7d25196..8abc81110 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -840,14 +840,17 @@ fn planner_comparisons_handle_nan_without_execution_errors() { } } -// A bounded LIMIT branch must unsubscribe so another branch can drain the producer. +// At ingestion time a LIMIT consumer of a fan-out node drops its read once it +// has its rows: the node's later batches are not queued for it, and the +// sibling reading the same node still gets every batch. #[test] -fn limit_branch_finishes_without_blocking_shared_sibling() { +fn limit_consumer_of_an_ingestion_fan_out_drops_its_read() { let input = schema(&[("v", DataType::Int64, false)]); let mut dag = PhysicalDAG::default(); - let batches = (0..100) + let batches: Vec<_> = (0..100) .map(|v| Batch::try_new(input.clone(), vec![vec![Value::Int64(v)]]).unwrap()) .collect(); + let batch_bytes = batches[0].memory_bytes(); dag.add( 0, vec![], @@ -869,29 +872,27 @@ fn limit_branch_finishes_without_blocking_shared_sibling() { Lifetime::None, ) .unwrap(); - // Bound polls as well as rows so a backpressure regression cannot hang the suite. - use futures::{task::noop_waker_ref, Stream}; - use std::{ - pin::Pin, - task::{Context, Poll}, - }; - let run = context(); - let mut stream = dag.execute(&[2], run.clone()).unwrap().remove(0); - let mut cx = Context::from_waker(noop_waker_ref()); - let mut count = 0; - for _ in 0..2000 { - match Pin::new(&mut stream).poll_next(&mut cx) { - Poll::Ready(Some(batch)) => count += batch.unwrap().to_rows().len(), - Poll::Ready(None) => { - assert_eq!(count, 101); - drop(stream); - assert_eq!(run.retained_bytes(), 0); - return; - } - Poll::Pending => {} - } - } - panic!("shared LIMIT/Union failed to make progress"); + let run = RunContext::new( + Scope::Ingestion { + window_start_ms: 0, + window_end_ms: 1, + }, + Limits::default(), + ) + .unwrap(); + let outputs = block_on(dag.execute_ingestion(&[2], run.clone()).unwrap()).unwrap(); + let rows: usize = outputs[0].iter().map(|batch| batch.num_rows()).sum(); + assert_eq!(rows, 101); + // The run holds the 101 output batches it returns and no more: had the + // LIMIT kept its read, its queue would hold the 99 source batches it + // never reads. + assert!( + run.peak_bytes() <= (101 + 2) * batch_bytes, + "{}", + run.peak_bytes() + ); + drop(outputs); + assert_eq!(run.retained_bytes(), 0); } // Mixed numeric comparisons must not round Int64 values through f64 before comparing. diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index e96d17d98..116f1686b 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -27,7 +27,7 @@ use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionT use crate::ir::properties::selection::SelectionBox; use crate::ir::schema::Schema; use crate::stages::PhysicalCandidateId; -use crate::stages::{Lifetime, PhysicalPropertiesError}; +use crate::stages::{Lifetime, PhysicalPropertiesError, StorageMedium}; use crate::summary_type::SketchParams; /// A node's operator, with each child replaced by its node id. @@ -677,8 +677,10 @@ impl PhysicalASAPDAG { /// no θ (every `parameters` is `None`), with each exported node's operator /// node by node id. A computed node with more than one consumer (edges and /// root positions) is held for one evaluation, as Stage 3 selects for a -/// shared query-time output; every other node is not materialized. Executor tests hand-build -/// DAGs this way; planning builds the plan in Stage 3. +/// shared query-time output; every other node is not materialized. Consumers +/// are counted per node, not per evaluation time, so a node read at two +/// times under an `@` anchor is conservatively held too. Executor tests +/// hand-build DAGs this way; planning builds the plan in Stage 3. #[cfg(any(test, feature = "test-support"))] pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( roots: &[crate::ir::QueryRoot], @@ -734,7 +736,7 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( } for node in &mut dag.nodes { if consumers[node.id] > 1 && computed[node.id] { - node.lifetime = Lifetime::OneEvaluation(crate::stages::StorageMedium::Memory); + node.lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); } } Ok((dag, nodes_by_id)) From a9d3ead637e6245e5579fb1a1708ac9d02430bc7 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:59:30 +0000 Subject: [PATCH 3/7] fix(executor): keep a node for later consumers, release on cancel, no partial held output Address the functional review: - An operator may drop its input in `start` before the node's next consumer subscribes; the stream is now dropped early only once every expected consumer has subscribed and left, instead of panicking later. - A cancellation seen by one producer cancels every producer of the run, releasing batches a held node keeps for consumers nobody polls. - A held node that fails partway clears its consumers' queues, so each gets only the error rather than a prefix that looks like success. --- crates/executor/src/runtime/mod.rs | 54 ++++++++++++++++--- crates/executor/src/runtime/tests.rs | 79 ++++++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 10 deletions(-) diff --git a/crates/executor/src/runtime/mod.rs b/crates/executor/src/runtime/mod.rs index 5bac8e1b5..379eb94ae 100644 --- a/crates/executor/src/runtime/mod.rs +++ b/crates/executor/src/runtime/mod.rs @@ -10,7 +10,7 @@ use std::{ collections::{BTreeMap, VecDeque}, fmt::Debug, pin::Pin, - rc::Rc, + rc::{Rc, Weak}, sync::Arc, task::{Context, Poll, Waker}, }; @@ -149,6 +149,7 @@ fn build<'r, V: 'r, S: 'r>( context: &RunContext, reached: &Reached, states: &mut BTreeMap>>>, + run_producers: &RunProducers<'r, V>, ) -> Result>>, Error> { let key = (id, context.evaluation_time_ms()); let node = &dag.nodes[&id]; @@ -167,6 +168,7 @@ fn build<'r, V: 'r, S: 'r>( input_context, reached, states, + run_producers, )?)); } let stream = node @@ -185,6 +187,11 @@ fn build<'r, V: 'r, S: 'r>( size: Box::new(move |value| op.output_bytes(value)), context: context.clone(), hold_until_finished: run == NodeRun::Held, + expected_readers: match run { + NodeRun::ReadPerConsumer => 1, + NodeRun::Held | NodeRun::Streamed => reached[&key].0, + }, + run_producers: Rc::clone(run_producers), unread: BTreeMap::new(), next_reader: 0, batches_polled: 0, @@ -192,6 +199,7 @@ fn build<'r, V: 'r, S: 'r>( finished: false, failure: None, })); + run_producers.borrow_mut().push(Rc::downgrade(&state)); if run != NodeRun::ReadPerConsumer { states.insert(key, Rc::clone(&state)); } @@ -222,11 +230,18 @@ fn start<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( } } let mut states = BTreeMap::new(); + let run_producers = RunProducers::default(); roots .iter() - .map(|&id| build(dag, id, &context, &reached, &mut states).map(Input::subscribe)) + .map(|&id| { + build(dag, id, &context, &reached, &mut states, &run_producers).map(Input::subscribe) + }) .collect() } +/// Every producer of one run, so a cancellation seen by one releases the +/// batches the others hold. +type RunProducers<'a, V> = Rc>>>>>; + /// One run of a node, read by every consumer it has in the run. struct Producer<'a, V> { node: NodeId, @@ -238,6 +253,11 @@ struct Producer<'a, V> { /// to the end before any consumer reads it, and its batches stay /// charged to the run until every consumer has read them. hold_until_finished: bool, + /// How many consumers subscribe to it in the run; its stream is dropped + /// early only once all have subscribed and all have left. + expected_readers: usize, + /// The run's producers, this one included. + run_producers: RunProducers<'a, V>, /// The batches each reader has not read yet, by reader. unread: BTreeMap>>, next_reader: u64, @@ -253,12 +273,25 @@ impl Producer<'_, V> { waker.wake(); } } + /// The end of the node's output. A failed held output is not read in + /// part: every consumer gets only the error. fn finish(&mut self, failure: Option) { + if failure.is_some() && self.hold_until_finished { + self.unread.values_mut().for_each(VecDeque::clear); + } self.failure = failure; self.finished = true; self.stream = None; self.wake_all(); } + /// Stop the run: every consumer gets `Cancelled` next, and the batches + /// not yet read are released. + fn cancel(&mut self) { + if self.failure != Some(Error::Cancelled) { + self.finish(Some(Error::Cancelled)); + self.unread.values_mut().for_each(VecDeque::clear); + } + } } pub struct Input<'a, V> { producer: Rc>>, @@ -286,7 +319,9 @@ impl Drop for Input<'_, V> { let mut state = self.producer.borrow_mut(); state.unread.remove(&self.reader); state.waiters.remove(&self.reader); - if state.unread.is_empty() { + // A consumer may drop its input in `start`, before a later consumer + // subscribes; the stream is kept for that one. + if state.unread.is_empty() && state.next_reader as usize >= state.expected_readers { state.stream = None; } } @@ -301,10 +336,15 @@ impl Stream for Input<'_, V> { let mut state = this.producer.borrow_mut(); state.context.register(cx.waker()); loop { - if state.context.is_cancelled() && !state.finished { - state.finish(Some(Error::Cancelled)); - for unread in state.unread.values_mut() { - unread.clear(); + if state.context.is_cancelled() && state.failure != Some(Error::Cancelled) { + state.cancel(); + // The others; a producer being polled up the stack sees the + // cancellation itself. + let run_producers = Rc::clone(&state.run_producers); + for producer in run_producers.borrow().iter().filter_map(Weak::upgrade) { + if let Ok(mut producer) = producer.try_borrow_mut() { + producer.cancel(); + } } } if state.finished || !state.hold_until_finished { diff --git a/crates/executor/src/runtime/tests.rs b/crates/executor/src/runtime/tests.rs index c9e2fd9a1..816923eb7 100644 --- a/crates/executor/src/runtime/tests.rs +++ b/crates/executor/src/runtime/tests.rs @@ -140,7 +140,9 @@ fn materialized_node_runs_once_and_is_held() { assert_eq!(polls.get(), 8); } -// Failure reaches every consumer; cancellation stops further producer work. +// Failure reaches every consumer, and a held node that fails partway is not +// read in part: each consumer gets only the error, so none reports success +// after its first batch. Cancellation stops further producer work. #[test] fn broadcast_error_and_cancel() { let (source, _, polls) = source(true); @@ -152,8 +154,8 @@ fn broadcast_error_and_cancel() { let b = outputs.pop().unwrap(); let (a, b) = block_on(async { futures::join!(a.collect::>(), b.collect::>()) }); for values in [a, b] { - assert_eq!(values.len(), 2); - assert!(matches!(values[1], Err(Error::AtNode { node: 0, .. }))); + assert_eq!(values.len(), 1); + assert!(matches!(values[0], Err(Error::AtNode { node: 0, .. }))); } assert_eq!(polls.get(), 2); let run = query_context(1024); @@ -473,3 +475,74 @@ fn one_evaluation_output_over_budget_is_a_memory_limit() { assert_eq!(run.retained_bytes(), 0); }); } + +// Ignores its input: drops it in `start` and emits nothing. +struct DropsInput; +impl PhysicalOperator for DropsInput { + fn name(&self) -> &str { + "DropsInput" + } + fn input_schemas(&self) -> Vec<()> { + vec![()] + } + fn output_schema(&self) {} + fn output_bytes(&self, _: &u64) -> usize { + 8 + } + fn start<'a>( + &'a self, + _: Vec>, + _: RunContext, + ) -> Result, Error> { + Ok(stream::empty().boxed_local()) + } +} + +// A consumer that drops its input in `start`, before the node's other +// consumer subscribes, leaves the node running for that one. +#[test] +fn an_input_dropped_at_start_leaves_the_node_to_later_consumers() { + let (source, _, _) = source(false); + let mut dag = PhysicalDAG::default(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); + dag.add(2, vec![1], DropsInput, Lifetime::None).unwrap(); + dag.add(3, vec![1], Identity, Lifetime::None).unwrap(); + let run = query_context(1024); + let outputs = dag.execute(&[2, 3], run.clone()).unwrap(); + let drained: Vec<_> = outputs + .into_iter() + .map(|output| values(block_on(output.collect::>()))) + .collect(); + assert_eq!(drained, vec![vec![], vec![Ok(0), Ok(1), Ok(2), Ok(3)]]); + assert_eq!(run.retained_bytes(), 0); +} + +// A cancellation seen by one root releases what a held node keeps for a +// root nobody polls, and every root then ends with `Cancelled`. +#[test] +fn cancellation_releases_held_batches_of_unpolled_roots() { + let (mut source, _, _) = source(false); + source.end = MANY_BATCHES; + let mut dag = PhysicalDAG::default(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); + dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); + dag.add(3, vec![1], Identity, Lifetime::None).unwrap(); + let run = query_context(1024); + let mut outputs = dag.execute(&[2, 3], run.clone()).unwrap(); + let first = block_on(outputs[0].next()).unwrap().unwrap(); + assert_eq!(*first, 0); + drop(first); + assert!(run.retained_bytes() > 0); + run.cancel(); + assert!(matches!( + block_on(outputs[0].next()), + Some(Err(Error::Cancelled)) + )); + assert_eq!(run.retained_bytes(), 0); + assert!(matches!( + block_on(outputs[1].next()), + Some(Err(Error::Cancelled)) + )); +} From 22998c93b5cf04971910beb5778787ab5825bc1a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:12:20 +0000 Subject: [PATCH 4/7] fix(executor): read a OneEvaluation scan once, hold only shared outputs Address the efficiency review: - A raw data slot carries its Scan node's plan lifetime. At query time a OneEvaluation scan with several consumers is read once and held for all of them, as the planner charges one read whose output is held; only a Lifetime::None scan (and a view slot, whose states the store keeps) is read once per consumer. - A materialized node is held only when it has more than one consumer at its evaluation time; with one consumer it streams, since no sibling can wait on that consumer. - README and runtime docs say which leaves are read per consumer, and that ingestion queues are bounded only by the byte budget. --- crates/executor/README.md | 35 +++--- .../executor/src/physical_planner/compiled.rs | 25 +++-- crates/executor/src/physical_planner/mod.rs | 1 + crates/executor/src/runtime/mod.rs | 44 +++++--- crates/executor/src/runtime/tests.rs | 101 ++++++++++++------ 5 files changed, 131 insertions(+), 75 deletions(-) diff --git a/crates/executor/README.md b/crates/executor/README.md index ed7f1140f..7cb44731e 100644 --- a/crates/executor/README.md +++ b/crates/executor/README.md @@ -14,22 +14,29 @@ worker-local streams, with no internal thread pool. A node runs once per evaluation time in a run (an `@` anchor evaluates the nodes below it at another time), as its `Lifetime` and the run's scope say: -- Query time, a leaf (a raw data or materialized view slot, a store read): - each consumer reads the store on its own, as the planner charges a - not-materialized read on each edge. -- Query time, `OneEvaluation` or `AcrossEvaluations`: the node runs to the - end before any consumer reads it, and every consumer reads its finished - output independently. The output is charged to the run's byte budget until - every consumer has read it; over the budget the run fails with - `MemoryLimit` (nothing spills). -- Query time, `None`: streamed to its one consumer. A computed node with more - than one consumer is a planner bug; `execute` returns an error naming the - node before any operator starts. +- Query time, `OneEvaluation` or `AcrossEvaluations`, with more than one + consumer (a computed node or a raw data scan): the node runs once, to the + end, before any consumer reads it, and every consumer reads its finished + output independently, as the planner charges one run whose output is held. + For example, a `OneEvaluation` scan read by three queries is one store + read. The output is charged to the run's byte budget until every consumer + has read it; over the budget the run fails with `MemoryLimit` (nothing + spills). With one consumer the node streams to it, since no sibling can + wait on that consumer. +- Query time, `None` leaf (a raw data scan the plan does not materialize, or + a materialized view slot, whose states the store keeps): each consumer + reads the store on its own, as the planner charges a not-materialized read + once per consumer. +- Query time, `None` computed node: streamed to its one consumer. More than + one consumer is a planner bug; `execute` returns an error naming the node + before any operator starts. - Ingestion time: every node streams, whatever its lifetime. A node with several consumers (one per-row node feeding several chain builds or views) - runs once and each batch reaches every consumer. `execute_ingestion` polls - every root itself and returns one future that resolves to each root's - batches, for the caller to write. + runs once and each batch reaches every consumer's own queue. The queues + are bounded only by the run's byte budget: a consumer that lags holds what + the others have read. `execute_ingestion` polls every root itself and + returns one future that resolves to each root's batches, for the caller to + write. `execute` runs a query and returns one stream per root; the caller may drain them in any order. It rejects an ingestion scope, which `execute_ingestion` diff --git a/crates/executor/src/physical_planner/compiled.rs b/crates/executor/src/physical_planner/compiled.rs index 198c997be..b595de5fe 100644 --- a/crates/executor/src/physical_planner/compiled.rs +++ b/crates/executor/src/physical_planner/compiled.rs @@ -44,6 +44,10 @@ pub(super) struct RawDataSlot { /// The rows its consumers read, back from the evaluation time; `None` /// when one reads every row. pub(super) read_window: Option, + /// The `Scan` node's plan lifetime, which the runtime runs the leaf by: + /// at query time `OneEvaluation` is one read held for every consumer, + /// `None` one read per consumer. + pub(super) lifetime: Lifetime, } /// Selected native operators and input slots. Rebinding never repeats lowering. @@ -170,10 +174,12 @@ impl CompiledPhysicalDAG { let mut dag = PhysicalDAG::default(); for (&id, node) in &self.nodes { match node { - Node::MaterializedView(MaterializedViewSlot { schema, .. }) - | Node::RawData(RawDataSlot { schema, .. }) => { - dag.add(id, vec![], UnboundInput(schema.clone()), SLOT_LIFETIME)? + Node::MaterializedView(MaterializedViewSlot { schema, .. }) => { + dag.add(id, vec![], UnboundInput(schema.clone()), VIEW_SLOT_LIFETIME)? } + Node::RawData(RawDataSlot { + schema, lifetime, .. + }) => dag.add(id, vec![], UnboundInput(schema.clone()), *lifetime)?, Node::Operator { inputs, operator, @@ -202,7 +208,7 @@ impl CompiledPhysicalDAG { slot.schema.clone(), slot.read_window, ); - dag.add(id, vec![], scan, SLOT_LIFETIME)?; + dag.add(id, vec![], scan, VIEW_SLOT_LIFETIME)?; } Node::RawData(slot) => { let scan = Scan::raw_data( @@ -212,7 +218,7 @@ impl CompiledPhysicalDAG { slot.predicates.clone(), slot.read_window, ); - dag.add(id, vec![], scan, SLOT_LIFETIME)?; + dag.add(id, vec![], scan, slot.lifetime)?; } Node::Operator { inputs, @@ -225,10 +231,11 @@ impl CompiledPhysicalDAG { Ok(dag) } } -/// An input slot's lifetime in the runtime DAG. The runtime runs a leaf by -/// the run's scope, not by its lifetime: at query time each consumer reads -/// the store, at ingestion time one read reaches every consumer. -const SLOT_LIFETIME: Lifetime = Lifetime::None; +/// A materialized view slot's lifetime in the runtime DAG. The store keeps +/// the view's states across evaluations, so the slot is a not-materialized +/// read of them: at query time each consumer reads the store, at ingestion +/// time one read reaches every consumer. +const VIEW_SLOT_LIFETIME: Lifetime = Lifetime::None; /// Stands in for an input slot when validating or deriving properties. struct UnboundInput(SchemaRef); diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 651e9b515..7c8ee327b 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -242,6 +242,7 @@ fn compile_internal( schema: output, predicates, read_window, + lifetime: node.lifetime, }, )?; } else { diff --git a/crates/executor/src/runtime/mod.rs b/crates/executor/src/runtime/mod.rs index 379eb94ae..b5ff76915 100644 --- a/crates/executor/src/runtime/mod.rs +++ b/crates/executor/src/runtime/mod.rs @@ -84,26 +84,34 @@ pub(crate) fn execute_ingestion<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( /// evaluates it at another time than the run's. type ProducerKey = (NodeId, Option); -/// How a node runs in a run, from the run's scope, whether it is a leaf and -/// its plan `Lifetime` (see the crate README). +/// How a node runs in a run, from the run's scope, its plan `Lifetime`, +/// whether it is a leaf and how many consumers it has at its evaluation +/// time (see the crate README). #[derive(Clone, Copy, PartialEq, Eq)] enum NodeRun { - /// Query time, a leaf (a raw data or materialized view slot): each - /// consumer reads the store on its own, as the planner charges a - /// not-materialized read on each edge. + /// Query time, a not-materialized leaf (a raw data or materialized view + /// slot): each consumer reads the store on its own, as the planner + /// charges a `Lifetime::None` read once per consumer. Example: a + /// `Lifetime::None` scan read by two queries is two store reads. ReadPerConsumer, - /// Query time, a materialized computed node: run to the end before any - /// consumer reads it, then read by each consumer. + /// Query time, a materialized node (computed or a leaf) with more than + /// one consumer: run to the end once before any consumer reads it, then + /// read by each consumer, as the planner charges one run whose output is + /// held. Example: a `OneEvaluation` scan read by three queries is one + /// store read whose batches every query reads. Held, /// Every other node: one run whose batches reach each consumer as they - /// come. At query time a computed one has a single consumer. + /// come. At query time it has a single consumer, or it is a computed + /// `Lifetime::None` node with several, which `start` rejects. Streamed, } -fn run_as(node: &Node<'_, V, S>, context: &RunContext) -> NodeRun { +fn run_as(node: &Node<'_, V, S>, context: &RunContext, consumers: usize) -> NodeRun { let query = context.evaluation_time_ms().is_some(); - match (query, node.inputs.is_empty(), node.lifetime) { - (true, true, _) => NodeRun::ReadPerConsumer, - (true, false, Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_)) => NodeRun::Held, + match (query, node.lifetime, node.inputs.is_empty()) { + (true, Lifetime::None, true) => NodeRun::ReadPerConsumer, + (true, Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), _) if consumers > 1 => { + NodeRun::Held + } _ => NodeRun::Streamed, } } @@ -153,7 +161,7 @@ fn build<'r, V: 'r, S: 'r>( ) -> Result>>, Error> { let key = (id, context.evaluation_time_ms()); let node = &dag.nodes[&id]; - let run = run_as(node, context); + let run = run_as(node, context, reached[&key].0); if run != NodeRun::ReadPerConsumer { if let Some(state) = states.get(&key) { return Ok(Rc::clone(state)); @@ -221,7 +229,8 @@ fn start<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( } for (&(id, time), (count, _)) in &reached { let node = &dag.nodes[&id]; - if let (Some(time), NodeRun::Streamed, 2..) = (time, run_as(node, &context), *count) { + if let (Some(time), NodeRun::Streamed, 2..) = (time, run_as(node, &context, *count), *count) + { return Err(Error::Invalid(format!( "node {id} ({}) at evaluation time {time} has {count} consumers but \ Lifetime::None; a shared query-time output must be OneEvaluation (planner bug)", @@ -249,9 +258,10 @@ struct Producer<'a, V> { stream: Option>, size: Box usize + 'a>, context: RunContext, - /// At query time, a materialized node ([`NodeRun::Held`]) is computed - /// to the end before any consumer reads it, and its batches stay - /// charged to the run until every consumer has read them. + /// At query time, a materialized node with several consumers + /// ([`NodeRun::Held`]) is computed to the end before any consumer reads + /// it, and its batches stay charged to the run until every consumer has + /// read them. hold_until_finished: bool, /// How many consumers subscribe to it in the run; its stream is dropped /// early only once all have subscribed and all have left. diff --git a/crates/executor/src/runtime/tests.rs b/crates/executor/src/runtime/tests.rs index 816923eb7..0b69a567f 100644 --- a/crates/executor/src/runtime/tests.rs +++ b/crates/executor/src/runtime/tests.rs @@ -89,8 +89,9 @@ fn source(fail: bool) -> (Source, Rc>, Rc>) { const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); -// At query time each consumer of a store read (a leaf) reads it on its own, -// so one consumer never waits for another. +// At query time each consumer of a not-materialized store read (a +// `Lifetime::None` leaf) reads it on its own, so one consumer never waits for +// another. #[test] fn query_time_leaves_are_read_per_consumer() { let (source, starts, polls) = source(false); @@ -313,11 +314,15 @@ fn within_timeout(run: impl FnOnce() -> T + Send + 'static) - /// More batches than any queue bound the runtime ever had (8 by default). const MANY_BATCHES: u64 = 20; -/// Node 0 emits `MANY_BATCHES` batches; nodes 1 and 2 each read it. -fn two_readers_of_one_leaf(dag: &mut PhysicalDAG<'static, u64, ()>) -> Rc> { +/// Node 0, with plan lifetime `leaf`, emits `MANY_BATCHES` batches; nodes 1 +/// and 2 each read it. +fn two_readers_of_one_leaf( + dag: &mut PhysicalDAG<'static, u64, ()>, + leaf: Lifetime, +) -> Rc> { let (mut source, starts, _) = source(false); source.end = MANY_BATCHES; - dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(0, vec![], source, leaf).unwrap(); dag.add(1, vec![0], Identity, Lifetime::None).unwrap(); dag.add(2, vec![0], Identity, Lifetime::None).unwrap(); starts @@ -340,44 +345,70 @@ fn all_batches() -> Vec> { } // Two queries read one stored view (y16): draining their roots one after -// another gives the same answers as polling them together. +// another gives the same answers as polling them together. A `None` leaf is +// read once per consumer (2 reads per run); a `OneEvaluation` leaf is read +// once and held for both (1 read per run). #[test] fn roots_reading_one_leaf_drain_in_order() { - within_timeout(|| { - let mut dag = PhysicalDAG::default(); - two_readers_of_one_leaf(&mut dag); - let outputs = dag.execute(&[1, 2], query_context(1024)).unwrap(); - let joined: Vec<_> = block_on(futures::future::join_all( - outputs.into_iter().map(|output| output.collect::>()), - )) - .into_iter() - .map(values) - .collect(); - assert_eq!(joined, vec![all_batches(), all_batches()]); - let run = query_context(1024); - let outputs = dag.execute(&[1, 2], run.clone()).unwrap(); - let drained: Vec<_> = outputs + for (leaf, reads_per_run) in [(Lifetime::None, 2), (ONE_EVALUATION, 1)] { + let reads = within_timeout(move || { + let mut dag = PhysicalDAG::default(); + let starts = two_readers_of_one_leaf(&mut dag, leaf); + let outputs = dag.execute(&[1, 2], query_context(1024)).unwrap(); + let joined: Vec<_> = block_on(futures::future::join_all( + outputs.into_iter().map(|output| output.collect::>()), + )) .into_iter() - .map(|output| values(block_on(output.collect::>()))) + .map(values) .collect(); - assert_eq!(drained, joined); - assert_eq!(run.retained_bytes(), 0); - }); + assert_eq!(joined, vec![all_batches(), all_batches()]); + let run = query_context(1024); + let outputs = dag.execute(&[1, 2], run.clone()).unwrap(); + let drained: Vec<_> = outputs + .into_iter() + .map(|output| values(block_on(output.collect::>()))) + .collect(); + assert_eq!(drained, joined); + assert_eq!(run.retained_bytes(), 0); + starts.get() + }); + assert_eq!(reads, 2 * reads_per_run, "{leaf:?}"); + } } // One query whose two branches read one stored view (b17, g10), combined by // an operator that reads its left branch to the end first. #[test] fn one_root_with_two_branches_over_one_leaf() { - within_timeout(|| { - let mut dag = PhysicalDAG::default(); - two_readers_of_one_leaf(&mut dag); - dag.add(3, vec![1, 2], LeftThenRight, Lifetime::None) - .unwrap(); - let output = dag.execute(&[3], query_context(1024)).unwrap().remove(0); - let both: Vec<_> = all_batches().into_iter().chain(all_batches()).collect(); - assert_eq!(values(block_on(output.collect::>())), both); - }); + for (leaf, reads) in [(Lifetime::None, 2), (ONE_EVALUATION, 1)] { + let starts = within_timeout(move || { + let mut dag = PhysicalDAG::default(); + let starts = two_readers_of_one_leaf(&mut dag, leaf); + dag.add(3, vec![1, 2], LeftThenRight, Lifetime::None) + .unwrap(); + let output = dag.execute(&[3], query_context(1024)).unwrap().remove(0); + let both: Vec<_> = all_batches().into_iter().chain(all_batches()).collect(); + assert_eq!(values(block_on(output.collect::>())), both); + starts.get() + }); + assert_eq!(starts, reads, "{leaf:?}"); + } +} + +// A materialized node with one consumer streams to it: no sibling can wait +// on that consumer, so nothing is held. Its first batch arrives after one +// source batch, not after the whole output. +#[test] +fn one_evaluation_node_with_one_consumer_streams() { + let (source, _, polls) = source(false); + let mut dag = PhysicalDAG::default(); + dag.add(0, vec![], source, ONE_EVALUATION).unwrap(); + dag.add(1, vec![0], Identity, ONE_EVALUATION).unwrap(); + let run = query_context(1024); + let mut output = dag.execute(&[1], run.clone()).unwrap().remove(0); + assert_eq!(*block_on(output.next()).unwrap().unwrap(), 0); + assert_eq!(polls.get(), 1); + assert_eq!(run.peak_bytes(), 8); } // At ingestion time one read of a source feeds several chain builds: the @@ -386,7 +417,7 @@ fn one_root_with_two_branches_over_one_leaf() { fn ingestion_fan_out_reads_the_source_once() { within_timeout(|| { let mut dag = PhysicalDAG::default(); - let starts = two_readers_of_one_leaf(&mut dag); + let starts = two_readers_of_one_leaf(&mut dag, Lifetime::None); dag.add(3, vec![0], Identity, Lifetime::None).unwrap(); dag.add(4, vec![1, 2], LeftThenRight, Lifetime::None) .unwrap(); @@ -407,7 +438,7 @@ fn ingestion_fan_out_reads_the_source_once() { #[test] fn execute_rejects_ingestion_scope_and_vice_versa() { let mut dag = PhysicalDAG::default(); - two_readers_of_one_leaf(&mut dag); + two_readers_of_one_leaf(&mut dag, Lifetime::None); let Err(error) = dag.execute(&[1], ingestion_context(1024)) else { panic!("`execute` ran an ingestion scope"); }; From fc8147269449eb1375e0b17df1dec4d57081caef Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:58:08 +0000 Subject: [PATCH 5/7] fix(executor): one stored-slot read rule, one lifetime map, no stream drop on cancel - A query-time leaf whose rows the store keeps (`None` or `AcrossEvaluations`) is read per consumer; a `OneEvaluation` node, or a computed `AcrossEvaluations` node, with more than one consumer is held. View slots carry their plan lifetime; the view-slot lifetime constant is gone. An `AcrossEvaluations` raw scan with several consumers is no longer held, which the planner never charged. - `CompiledPhysicalDAG` keeps every Planner node's lifetime in one map; `RawDataSlot::lifetime`, the operator node's lifetime and the extra lowering argument are gone, and `from_operators` takes main's pairs. - Cancelling the other producers of a run no longer drops their streams while the producer that saw the cancellation is borrowed, which panicked with "already borrowed" when an operator cancelled inside its own poll. - The fan-out check states its condition directly; `ReachedNode` names the walk's per-node facts; `Producer::consumers`; `IngestionOutputs` is the batches, used inside `Result`. - The query-time test helper marks every shared node `OneEvaluation`, leaves included; `views_at` makes a `OneEvaluation` node it moves to ingestion time `None`. - Docs: the read rule lives in `NodeRun` and the README; the design doc's slot table has a lifetime column; the executing-a-plan tutorial names `execute_ingestion` and `execute`. --- crates/executor/README.md | 30 ++-- .../executor/src/physical_planner/compiled.rs | 75 ++++------ crates/executor/src/physical_planner/mod.rs | 85 ++++------- .../src/physical_planner/promql_values.rs | 5 +- .../src/physical_planner/test_support.rs | 8 +- crates/executor/src/plan/mod.rs | 5 +- crates/executor/src/runtime/mod.rs | 111 ++++++++------ crates/executor/src/runtime/tests.rs | 136 ++++++++++++++++-- crates/executor/tests/current_series_heap.rs | 11 +- crates/executor/tests/hydra_cms_execution.rs | 9 +- crates/executor/tests/physical_dag.rs | 15 +- crates/executor/tests/promql_binary.rs | 7 +- crates/executor/tests/promql_values.rs | 10 +- .../src/tutorials/_05_executing_a_plan.md | 12 +- crates/types/src/plan/physical_asap_dag.rs | 20 ++- crates/types/src/test_support/forwarders.rs | 6 +- .../physical-planning-and-deployment.md | 13 +- 17 files changed, 321 insertions(+), 237 deletions(-) diff --git a/crates/executor/README.md b/crates/executor/README.md index 7cb44731e..7c9c3a019 100644 --- a/crates/executor/README.md +++ b/crates/executor/README.md @@ -14,19 +14,23 @@ worker-local streams, with no internal thread pool. A node runs once per evaluation time in a run (an `@` anchor evaluates the nodes below it at another time), as its `Lifetime` and the run's scope say: -- Query time, `OneEvaluation` or `AcrossEvaluations`, with more than one - consumer (a computed node or a raw data scan): the node runs once, to the - end, before any consumer reads it, and every consumer reads its finished - output independently, as the planner charges one run whose output is held. - For example, a `OneEvaluation` scan read by three queries is one store - read. The output is charged to the run's byte budget until every consumer - has read it; over the budget the run fails with `MemoryLimit` (nothing - spills). With one consumer the node streams to it, since no sibling can - wait on that consumer. -- Query time, `None` leaf (a raw data scan the plan does not materialize, or - a materialized view slot, whose states the store keeps): each consumer - reads the store on its own, as the planner charges a not-materialized read - once per consumer. +- Query time, a leaf whose rows the store keeps: `None` (a raw data scan the + plan does not materialize, or any other leaf such as a constant) or + `AcrossEvaluations` (a materialized view slot, or raw data kept at + ingestion). Each consumer reads it from the store on its own, and + nothing is held. Stage 3 never gives a query-time `None` result more than + one consumer; a view slot, or a leaf in a hand-built DAG, may have + several. For example, an `AcrossEvaluations` scan read by two queries is + two store reads. +- Query time, `OneEvaluation` (a computed node or a leaf), or a computed + `AcrossEvaluations` node, with more than one consumer: the node runs once, + to the end, before any consumer reads it, and every consumer reads its + finished output independently, as the planner charges one run whose + output is held. For example, a `OneEvaluation` scan read by three queries + is one store read. The output is charged to the run's byte budget until + every consumer has read it; over the budget the run fails with + `MemoryLimit` (nothing spills). With one consumer the node streams to it, + since no sibling can wait on that consumer. - Query time, `None` computed node: streamed to its one consumer. More than one consumer is a planner bug; `execute` returns an error naming the node before any operator starts. diff --git a/crates/executor/src/physical_planner/compiled.rs b/crates/executor/src/physical_planner/compiled.rs index b595de5fe..369c330e1 100644 --- a/crates/executor/src/physical_planner/compiled.rs +++ b/crates/executor/src/physical_planner/compiled.rs @@ -15,7 +15,6 @@ enum Node { Operator { inputs: Vec, operator: Operator, - lifetime: Lifetime, }, } @@ -44,10 +43,6 @@ pub(super) struct RawDataSlot { /// The rows its consumers read, back from the evaluation time; `None` /// when one reads every row. pub(super) read_window: Option, - /// The `Scan` node's plan lifetime, which the runtime runs the leaf by: - /// at query time `OneEvaluation` is one read held for every consumer, - /// `None` one read per consumer. - pub(super) lifetime: Lifetime, } /// Selected native operators and input slots. Rebinding never repeats lowering. @@ -58,26 +53,33 @@ pub(super) struct RawDataSlot { pub struct CompiledPhysicalDAG { nodes: BTreeMap, roots: Vec, + /// The plan lifetime of each Planner node, which the runtime runs it by + /// (see `NodeRun` in the runtime). A helper node the lowering adds is + /// not in it and has `Lifetime::None`. Example: node 1, a stored view, + /// maps to `AcrossEvaluations(Memory)`, and node 3, a query-time output + /// two queries read, to `OneEvaluation(Memory)`. + lifetimes: BTreeMap, } impl CompiledPhysicalDAG { /// Assemble already-lowered operators; leaves read no data store (for /// example [`Operator::source`] holds its rows). pub fn from_operators( - operators: BTreeMap, Operator, Lifetime)>, + operators: BTreeMap, Operator)>, roots: Vec, ) -> Result { - let mut result = Self::new(roots); - for (id, (inputs, operator, lifetime)) in operators { - result.add(id, inputs, operator, lifetime)?; + let mut result = Self::new(roots, BTreeMap::new()); + for (id, (inputs, operator)) in operators { + result.add(id, inputs, operator)?; } result.validate()?; Ok(result) } - pub(super) fn new(roots: Vec) -> Self { + pub(super) fn new(roots: Vec, lifetimes: BTreeMap) -> Self { Self { nodes: BTreeMap::new(), roots, + lifetimes, } } pub(super) fn add_materialized_view( @@ -95,16 +97,8 @@ impl CompiledPhysicalDAG { id: NodeId, inputs: Vec, operator: Operator, - lifetime: Lifetime, ) -> Result<(), Error> { - self.insert( - id, - Node::Operator { - inputs, - operator, - lifetime, - }, - ) + self.insert(id, Node::Operator { inputs, operator }) } fn insert(&mut self, id: NodeId, node: Node) -> Result<(), Error> { if self.nodes.insert(id, node).is_some() { @@ -117,9 +111,7 @@ impl CompiledPhysicalDAG { pub fn row_source(&self, id: NodeId) -> Option { match self.nodes.get(&id)? { Node::MaterializedView(_) | Node::RawData(_) => Some(id), - Node::Operator { - inputs, operator, .. - } => { + Node::Operator { inputs, operator } => { let index = operator.row_preserving_input()?; self.row_source(*inputs.get(index)?) } @@ -173,18 +165,15 @@ impl CompiledPhysicalDAG { fn unbound(&self) -> Result, Error> { let mut dag = PhysicalDAG::default(); for (&id, node) in &self.nodes { + let lifetime = self.lifetime(id); match node { - Node::MaterializedView(MaterializedViewSlot { schema, .. }) => { - dag.add(id, vec![], UnboundInput(schema.clone()), VIEW_SLOT_LIFETIME)? + Node::MaterializedView(MaterializedViewSlot { schema, .. }) + | Node::RawData(RawDataSlot { schema, .. }) => { + dag.add(id, vec![], UnboundInput(schema.clone()), lifetime)? + } + Node::Operator { inputs, operator } => { + dag.add(id, inputs.clone(), operator.clone(), lifetime)? } - Node::RawData(RawDataSlot { - schema, lifetime, .. - }) => dag.add(id, vec![], UnboundInput(schema.clone()), *lifetime)?, - Node::Operator { - inputs, - operator, - lifetime, - } => dag.add(id, inputs.clone(), operator.clone(), *lifetime)?, } } dag.validate(&self.roots)?; @@ -200,6 +189,7 @@ impl CompiledPhysicalDAG { ) -> Result, Error> { let mut dag = PhysicalDAG::default(); for (&id, node) in &self.nodes { + let lifetime = self.lifetime(id); match node { Node::MaterializedView(slot) => { let scan = Scan::materialized_view( @@ -208,7 +198,7 @@ impl CompiledPhysicalDAG { slot.schema.clone(), slot.read_window, ); - dag.add(id, vec![], scan, VIEW_SLOT_LIFETIME)?; + dag.add(id, vec![], scan, lifetime)?; } Node::RawData(slot) => { let scan = Scan::raw_data( @@ -218,25 +208,20 @@ impl CompiledPhysicalDAG { slot.predicates.clone(), slot.read_window, ); - dag.add(id, vec![], scan, slot.lifetime)?; + dag.add(id, vec![], scan, lifetime)?; + } + Node::Operator { inputs, operator } => { + dag.add(id, inputs.clone(), operator.clone(), lifetime)? } - Node::Operator { - inputs, - operator, - lifetime, - } => dag.add(id, inputs.clone(), operator.clone(), *lifetime)?, } } dag.validate(&self.roots)?; Ok(dag) } + fn lifetime(&self, id: NodeId) -> Lifetime { + self.lifetimes.get(&id).copied().unwrap_or_default() + } } -/// A materialized view slot's lifetime in the runtime DAG. The store keeps -/// the view's states across evaluations, so the slot is a not-materialized -/// read of them: at query time each consumer reads the store, at ingestion -/// time one read reaches every consumer. -const VIEW_SLOT_LIFETIME: Lifetime = Lifetime::None; - /// Stands in for an input slot when validating or deriving properties. struct UnboundInput(SchemaRef); impl PhysicalOperator for UnboundInput { diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 7c8ee327b..0b4809622 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -21,7 +21,6 @@ use planner_types::plan::physical_asap_dag::{ PhysicalASAPOperatorPayload as Payload, }; use planner_types::plan::TimeWindow; -use planner_types::stages::Lifetime; use planner_types::summary_type::{ SketchParams, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, }; @@ -184,7 +183,11 @@ fn compile_internal( consumers: &consumers, roots: &compile_roots, }; - let mut physical_dag = CompiledPhysicalDAG::new(roots.to_vec()); + let lifetimes = nodes + .iter() + .map(|(&id, node)| (id, node.lifetime)) + .collect(); + let mut physical_dag = CompiledPhysicalDAG::new(roots.to_vec(), lifetimes); for id in ordered { let node = nodes[&id]; let auxiliary = helper_id(id, 0)?; @@ -242,7 +245,6 @@ fn compile_internal( schema: output, predicates, read_window, - lifetime: node.lifetime, }, )?; } else { @@ -263,7 +265,6 @@ fn compile_internal( auxiliary, inputs, Operator::union(schemas[0].clone(), schemas.len())?, - Lifetime::None, )?; inputs = vec![auxiliary]; schemas.truncate(1); @@ -361,7 +362,6 @@ fn compile_internal( inputs, Operator::current_series(input.clone(), identity, coordinate, value, lookback)? .with_output_schema(output)?, - node.lifetime, )?; continue; } @@ -395,15 +395,10 @@ fn compile_internal( let mut inputs = inputs; for (index, operator) in chain.into_iter().enumerate() { let helper = helper_id(id, index as u64)?; - physical_dag.add(helper, inputs, operator, Lifetime::None)?; + physical_dag.add(helper, inputs, operator)?; inputs = vec![helper]; } - physical_dag.add( - id, - inputs, - last.with_output_schema(output)?, - node.lifetime, - )?; + physical_dag.add(id, inputs, last.with_output_schema(output)?)?; continue; }; let groups = spec @@ -416,7 +411,6 @@ fn compile_internal( id, inputs, top_k_sort(&input, value, groups, *k as u64)?.with_output_schema(output)?, - node.lifetime, )?; continue; } @@ -498,8 +492,8 @@ fn compile_internal( .collect(), )?; let projected = project.schema(); - physical_dag.add(auxiliary, inputs, aggregate, Lifetime::None)?; - physical_dag.add(helper_id(id, 1)?, vec![auxiliary], project, Lifetime::None)?; + physical_dag.add(auxiliary, inputs, aggregate)?; + physical_dag.add(helper_id(id, 1)?, vec![auxiliary], project)?; let stamped = Operator::scope_timestamp(projected, output.clone())?; if output .fields @@ -508,20 +502,14 @@ fn compile_internal( { // Range functions drop the metric name, as their // summary evaluations do. - physical_dag.add( - helper_id(id, 2)?, - vec![helper_id(id, 1)?], - stamped, - Lifetime::None, - )?; + physical_dag.add(helper_id(id, 2)?, vec![helper_id(id, 1)?], stamped)?; physical_dag.add( id, vec![helper_id(id, 2)?], Operator::series_without_name(output)?, - node.lifetime, )?; } else { - physical_dag.add(id, vec![helper_id(id, 1)?], stamped, node.lifetime)?; + physical_dag.add(id, vec![helper_id(id, 1)?], stamped)?; } continue; } @@ -653,13 +641,12 @@ fn compile_internal( .ok_or_else(|| invalid("a chain's build drops its window end")) }) .transpose()?; - physical_dag.add(auxiliary, inputs, window, Lifetime::None)?; - physical_dag.add(helper_id(id, 1)?, vec![auxiliary], build, Lifetime::None)?; + physical_dag.add(auxiliary, inputs, window)?; + physical_dag.add(helper_id(id, 1)?, vec![auxiliary], build)?; physical_dag.add( id, vec![helper_id(id, 1)?], Operator::window_end_timestamp(compact, output, window_end)?, - node.lifetime, )?; continue; } @@ -682,7 +669,6 @@ fn compile_internal( inputs, Operator::evaluate_at(input.clone(), *at, *offset_ms)? .with_output_schema(output)?, - node.lifetime, )?; continue; } @@ -751,12 +737,11 @@ fn compile_internal( groups, )?; let compact = build.schema(); - physical_dag.add(auxiliary, inputs, build, Lifetime::None)?; + physical_dag.add(auxiliary, inputs, build)?; physical_dag.add( id, vec![auxiliary], Operator::scope_timestamp(compact, output)?, - node.lifetime, )?; continue; } @@ -789,12 +774,7 @@ fn compile_internal( [false, false], ) .map_err(|error| invalid(format!("node {id}: {error}")))?; - physical_dag.add( - id, - inputs, - binary.with_output_schema(output)?, - node.lifetime, - )?; + physical_dag.add(id, inputs, binary.with_output_schema(output)?)?; continue; } } @@ -826,22 +806,16 @@ fn compile_internal( .collect(); let project = Operator::project(actual, columns)?.with_output_schema(output.clone())?; - physical_dag.add(auxiliary, inputs, evaluation, Lifetime::None)?; + physical_dag.add(auxiliary, inputs, evaluation)?; if temporal_evaluation_drops_name(node) { - physical_dag.add( - auxiliary - 1, - vec![auxiliary], - project, - Lifetime::None, - )?; + physical_dag.add(auxiliary - 1, vec![auxiliary], project)?; physical_dag.add( id, vec![auxiliary - 1], Operator::series_without_name(output)?, - node.lifetime, )?; } else { - physical_dag.add(id, vec![auxiliary], project, node.lifetime)?; + physical_dag.add(id, vec![auxiliary], project)?; } continue; } @@ -858,12 +832,11 @@ fn compile_internal( // as a per-series `SummaryAgg` does. let merged = bind_operation(node, &schemas)?; let compact = merged.schema(); - physical_dag.add(helper_id(id, 1)?, inputs, merged, Lifetime::None)?; + physical_dag.add(helper_id(id, 1)?, inputs, merged)?; physical_dag.add( id, vec![helper_id(id, 1)?], Operator::scope_timestamp(compact, output)?, - node.lifetime, )?; continue; } @@ -881,7 +854,7 @@ fn compile_internal( let operator = compile_node(node, &schemas) .and_then(|merge| merge.with_emit_empty_global(parameters)) .map_err(|error| invalid(format!("node {id}: {error}")))?; - physical_dag.add(id, inputs, operator, node.lifetime)?; + physical_dag.add(id, inputs, operator)?; continue; } } @@ -890,12 +863,7 @@ fn compile_internal( if let Some(operator) = time_window_operator(id, &graph, &schemas) .map_err(|error| invalid(format!("node {id}: {error}")))? { - physical_dag.add( - id, - inputs, - operator.with_output_schema(output)?, - node.lifetime, - )?; + physical_dag.add(id, inputs, operator.with_output_schema(output)?)?; continue; } let mut operator = compile_node(node, &schemas) @@ -906,15 +874,10 @@ fn compile_internal( } } if temporal_evaluation_drops_name(node) { - physical_dag.add(auxiliary, inputs, operator, Lifetime::None)?; - physical_dag.add( - id, - vec![auxiliary], - Operator::series_without_name(output)?, - node.lifetime, - )?; + physical_dag.add(auxiliary, inputs, operator)?; + physical_dag.add(id, vec![auxiliary], Operator::series_without_name(output)?)?; } else { - physical_dag.add(id, inputs, operator, node.lifetime)?; + physical_dag.add(id, inputs, operator)?; } } } diff --git a/crates/executor/src/physical_planner/promql_values.rs b/crates/executor/src/physical_planner/promql_values.rs index a60938078..12a79a4d6 100644 --- a/crates/executor/src/physical_planner/promql_values.rs +++ b/crates/executor/src/physical_planner/promql_values.rs @@ -18,8 +18,5 @@ pub fn compile_scalar(value: f64) -> Result { planner_types::ir::schema::DataType::Float64, )? .with_output_schema(scalar_schema())?; - CompiledPhysicalDAG::from_operators( - BTreeMap::from([(0, (vec![], operator, Lifetime::None))]), - vec![0], - ) + CompiledPhysicalDAG::from_operators(BTreeMap::from([(0, (vec![], operator))]), vec![0]) } diff --git a/crates/executor/src/physical_planner/test_support.rs b/crates/executor/src/physical_planner/test_support.rs index fb288a546..945b50eb8 100644 --- a/crates/executor/src/physical_planner/test_support.rs +++ b/crates/executor/src/physical_planner/test_support.rs @@ -37,7 +37,10 @@ pub fn set_build_parameters<'a>( /// The materialized views at `nodes`, in that order: each node that is /// not a view is marked materialized at ingestion time, and it and /// every node below it run at ingestion time (their outputs and -/// outgoing edges); every view without a retention or coverage is +/// outgoing edges). A `OneEvaluation` node moved there becomes +/// `Lifetime::None`, as ingestion has no evaluation to hold it for: for +/// example a scan five tumbling windows share streams to all five. Every +/// view without a retention or coverage is /// retained for ever with a coverage of its own. For tests that bind /// states at those nodes, as a deployment binds its views. pub fn views_at( @@ -87,6 +90,9 @@ pub fn views_at( } if at_ingestion.contains(&id) { node.output_state.timing = ExecutionTiming::IngestionTime; + if matches!(node.lifetime, Lifetime::OneEvaluation(_)) { + node.lifetime = Lifetime::None; + } } } for edge in &mut dag.edges { diff --git a/crates/executor/src/plan/mod.rs b/crates/executor/src/plan/mod.rs index 0f868074d..71884a244 100644 --- a/crates/executor/src/plan/mod.rs +++ b/crates/executor/src/plan/mod.rs @@ -49,7 +49,8 @@ pub(crate) struct Node<'a, V, S> { pub(crate) inputs: Vec, pub(crate) operator: Box + 'a>, /// How long its output is kept, from the plan: what the runtime runs it - /// by (see the crate README). + /// by (see the crate README). Example: `OneEvaluation`, computed once + /// and held for its consumers at query time. pub(crate) lifetime: Lifetime, } pub struct PhysicalDAG<'a, V, S> { @@ -190,7 +191,7 @@ impl<'a, V: 'a, S: Clone + PartialEq + Debug + 'a> PhysicalDAG<'a, V, S> { &'r self, roots: &[NodeId], context: RunContext, - ) -> Result> + 'r, Error> + ) -> Result, Error>> + 'r, Error> where 'a: 'r, { diff --git a/crates/executor/src/runtime/mod.rs b/crates/executor/src/runtime/mod.rs index b5ff76915..ad51a540e 100644 --- a/crates/executor/src/runtime/mod.rs +++ b/crates/executor/src/runtime/mod.rs @@ -18,7 +18,7 @@ mod context; pub use context::{Limits, Reservation, RunContext, Scope}; pub type OutputStream<'a, V> = LocalBoxStream<'a, Result>; /// Every root's batches of an ingestion run, in the order of its roots. -pub type IngestionOutputs = Result>>, Error>; +pub type IngestionOutputs = Vec>>; /// An output owns its memory reservation even after it leaves the DAG's queue. pub struct SharedValue { value: Arc, @@ -68,7 +68,7 @@ pub(crate) fn execute_ingestion<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( dag: &'r PhysicalDAG<'_, V, S>, roots: &[NodeId], context: RunContext, -) -> Result> + 'r, Error> { +) -> Result, Error>> + 'r, Error> { if context.evaluation_time_ms().is_some() { return Err(Error::Invalid( "`execute_ingestion` takes an ingestion scope; run a query scope with `execute`".into(), @@ -86,29 +86,34 @@ type ProducerKey = (NodeId, Option); /// How a node runs in a run, from the run's scope, its plan `Lifetime`, /// whether it is a leaf and how many consumers it has at its evaluation -/// time (see the crate README). +/// time. This is the one statement of the rule; the crate README describes +/// it for callers. #[derive(Clone, Copy, PartialEq, Eq)] enum NodeRun { - /// Query time, a not-materialized leaf (a raw data or materialized view - /// slot): each consumer reads the store on its own, as the planner - /// charges a `Lifetime::None` read once per consumer. Example: a - /// `Lifetime::None` scan read by two queries is two store reads. + /// Query time, a leaf whose rows the store keeps: `Lifetime::None` (a + /// raw data slot, or any other leaf such as a constant) or + /// `AcrossEvaluations` (a materialized view slot, or raw data kept at + /// ingestion). Each consumer reads it on its own and nothing is held. + /// Stage 3 never gives a query-time `None` result more than one + /// consumer; a view slot, or a leaf in a hand-built DAG, may have + /// several. Example: an `AcrossEvaluations` scan read by two queries is + /// two store reads. ReadPerConsumer, - /// Query time, a materialized node (computed or a leaf) with more than - /// one consumer: run to the end once before any consumer reads it, then - /// read by each consumer, as the planner charges one run whose output is - /// held. Example: a `OneEvaluation` scan read by three queries is one - /// store read whose batches every query reads. + /// Query time, a `OneEvaluation` node (computed or a leaf), or a computed + /// `AcrossEvaluations` node, with more than one consumer: run to the end + /// once before any consumer reads it, then read by each consumer, as the + /// planner charges one run whose output is held. Example: a + /// `OneEvaluation` scan read by three queries is one store read whose + /// batches every query reads. Held, /// Every other node: one run whose batches reach each consumer as they - /// come. At query time it has a single consumer, or it is a computed - /// `Lifetime::None` node with several, which `start` rejects. + /// come. At query time it has one consumer (a computed `None` node with + /// several is a planner bug, which `start` rejects). Streamed, } -fn run_as(node: &Node<'_, V, S>, context: &RunContext, consumers: usize) -> NodeRun { - let query = context.evaluation_time_ms().is_some(); +fn run_as(node: &Node<'_, V, S>, query: bool, consumers: usize) -> NodeRun { match (query, node.lifetime, node.inputs.is_empty()) { - (true, Lifetime::None, true) => NodeRun::ReadPerConsumer, + (true, Lifetime::None | Lifetime::AcrossEvaluations(_), true) => NodeRun::ReadPerConsumer, (true, Lifetime::OneEvaluation(_) | Lifetime::AcrossEvaluations(_), _) if consumers > 1 => { NodeRun::Held } @@ -116,9 +121,15 @@ fn run_as(node: &Node<'_, V, S>, context: &RunContext, consumers: usize) - } } -/// Each (node, evaluation time) a run reaches: its consumer count and the -/// context its inputs run in. -type Reached = BTreeMap; +/// A (node, evaluation time) a run reaches. Example: a scan read by both +/// sides of a division at time T has `consumers: 2`, and `input_context` is +/// T's context (another time's under an `@` anchor). +struct ReachedNode { + consumers: usize, + /// The run context its inputs run in. + input_context: RunContext, +} +type Reached = BTreeMap; /// Visit `id` once per evaluation time, before any operator starts: /// validate its run context, count its consumers and record its inputs' @@ -130,8 +141,8 @@ fn reach( reached: &mut Reached, ) -> Result<(), Error> { let key = (id, context.evaluation_time_ms()); - if let Some((count, _)) = reached.get_mut(&key) { - *count += 1; + if let Some(reached) = reached.get_mut(&key) { + reached.consumers += 1; return Ok(()); } let node = &dag.nodes[&id]; @@ -147,7 +158,13 @@ fn reach( for &child in &node.inputs { reach(dag, child, &input_context, reached)?; } - reached.insert(key, (1, input_context)); + reached.insert( + key, + ReachedNode { + consumers: 1, + input_context, + }, + ); Ok(()) } @@ -161,13 +178,14 @@ fn build<'r, V: 'r, S: 'r>( ) -> Result>>, Error> { let key = (id, context.evaluation_time_ms()); let node = &dag.nodes[&id]; - let run = run_as(node, context, reached[&key].0); + let reached_node = &reached[&key]; + let run = run_as(node, key.1.is_some(), reached_node.consumers); if run != NodeRun::ReadPerConsumer { if let Some(state) = states.get(&key) { return Ok(Rc::clone(state)); } } - let input_context = &reached[&key].1; + let input_context = &reached_node.input_context; let mut inputs = Vec::new(); for &child in &node.inputs { inputs.push(Input::subscribe(build( @@ -195,9 +213,9 @@ fn build<'r, V: 'r, S: 'r>( size: Box::new(move |value| op.output_bytes(value)), context: context.clone(), hold_until_finished: run == NodeRun::Held, - expected_readers: match run { + consumers: match run { NodeRun::ReadPerConsumer => 1, - NodeRun::Held | NodeRun::Streamed => reached[&key].0, + NodeRun::Held | NodeRun::Streamed => reached_node.consumers, }, run_producers: Rc::clone(run_producers), unread: BTreeMap::new(), @@ -227,12 +245,13 @@ fn start<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( for &root in roots { reach(dag, root, &context, &mut reached)?; } - for (&(id, time), (count, _)) in &reached { + for (&(id, time), ReachedNode { consumers, .. }) in &reached { let node = &dag.nodes[&id]; - if let (Some(time), NodeRun::Streamed, 2..) = (time, run_as(node, &context, *count), *count) + if let (Some(time), Lifetime::None, false, 2..) = + (time, node.lifetime, node.inputs.is_empty(), *consumers) { return Err(Error::Invalid(format!( - "node {id} ({}) at evaluation time {time} has {count} consumers but \ + "node {id} ({}) at evaluation time {time} has {consumers} consumers but \ Lifetime::None; a shared query-time output must be OneEvaluation (planner bug)", node.operator.name() ))); @@ -265,7 +284,7 @@ struct Producer<'a, V> { hold_until_finished: bool, /// How many consumers subscribe to it in the run; its stream is dropped /// early only once all have subscribed and all have left. - expected_readers: usize, + consumers: usize, /// The run's producers, this one included. run_producers: RunProducers<'a, V>, /// The batches each reader has not read yet, by reader. @@ -283,24 +302,32 @@ impl Producer<'_, V> { waker.wake(); } } - /// The end of the node's output. A failed held output is not read in - /// part: every consumer gets only the error. + /// The end of the node's output, seen while polling its stream, which + /// is dropped. fn finish(&mut self, failure: Option) { - if failure.is_some() && self.hold_until_finished { - self.unread.values_mut().for_each(VecDeque::clear); - } - self.failure = failure; - self.finished = true; + self.end(failure); self.stream = None; - self.wake_all(); } /// Stop the run: every consumer gets `Cancelled` next, and the batches - /// not yet read are released. + /// not yet read are released. The stream is kept, and dropped with the + /// node's last input: the producer that saw the cancellation may be + /// borrowed up the poll stack, and an input inside this stream, of + /// another consumer of it, would borrow it again on drop. fn cancel(&mut self) { if self.failure != Some(Error::Cancelled) { - self.finish(Some(Error::Cancelled)); + self.end(Some(Error::Cancelled)); + } + } + /// Every consumer gets `failure`, or the end, next. A failed held output + /// is not read in part: every consumer gets only the error. A cancelled + /// output, held or not, releases every batch not yet read. + fn end(&mut self, failure: Option) { + if (failure.is_some() && self.hold_until_finished) || failure == Some(Error::Cancelled) { self.unread.values_mut().for_each(VecDeque::clear); } + self.failure = failure; + self.finished = true; + self.wake_all(); } } pub struct Input<'a, V> { @@ -331,7 +358,7 @@ impl Drop for Input<'_, V> { state.waiters.remove(&self.reader); // A consumer may drop its input in `start`, before a later consumer // subscribes; the stream is kept for that one. - if state.unread.is_empty() && state.next_reader as usize >= state.expected_readers { + if state.unread.is_empty() && state.next_reader as usize >= state.consumers { state.stream = None; } } diff --git a/crates/executor/src/runtime/tests.rs b/crates/executor/src/runtime/tests.rs index 0b69a567f..7f40d940f 100644 --- a/crates/executor/src/runtime/tests.rs +++ b/crates/executor/src/runtime/tests.rs @@ -93,7 +93,7 @@ const ONE_EVALUATION: Lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); // `Lifetime::None` leaf) reads it on its own, so one consumer never waits for // another. #[test] -fn query_time_leaves_are_read_per_consumer() { +fn not_materialized_leaf_is_read_per_consumer() { let (source, starts, polls) = source(false); let mut dag = PhysicalDAG::default(); dag.add(0, vec![], source, Lifetime::None).unwrap(); @@ -113,6 +113,42 @@ fn query_time_leaves_are_read_per_consumer() { assert_eq!(context.retained_bytes(), 0); } +// A leaf whose rows the store keeps across evaluations (an +// `AcrossEvaluations` raw data scan) is read from the store by each +// consumer on its own: nothing is held in the run. Example: two queries over +// one retained scan are two store reads, and the first query's first batch +// is the only one charged while it reads it. +#[test] +fn stored_leaf_is_read_per_consumer() { + let (source, starts, polls) = source(false); + let mut dag = PhysicalDAG::default(); + dag.add( + 0, + vec![], + source, + Lifetime::AcrossEvaluations(StorageMedium::Memory), + ) + .unwrap(); + let context = query_context(1024); + let mut readers = dag.execute(&[0, 0], context.clone()).unwrap(); + let second = readers.pop().unwrap(); + let mut first = readers.pop().unwrap(); + let batch = block_on(first.next()).unwrap().unwrap(); + assert_eq!(*batch, 0); + assert_eq!(context.retained_bytes(), 8); + drop(batch); + assert_eq!( + values(block_on(first.collect::>())), + vec![Ok(1), Ok(2), Ok(3)] + ); + assert_eq!( + values(block_on(second.collect::>())), + vec![Ok(0), Ok(1), Ok(2), Ok(3)] + ); + assert_eq!((starts.get(), polls.get()), (2, 8)); + assert_eq!(context.retained_bytes(), 0); +} + // A materialized node runs once per run, to the end, and every consumer // reads its finished output, charged to the run until all have read it. #[test] @@ -344,13 +380,18 @@ fn all_batches() -> Vec> { (0..MANY_BATCHES).map(Ok).collect() } -// Two queries read one stored view (y16): draining their roots one after -// another gives the same answers as polling them together. A `None` leaf is -// read once per consumer (2 reads per run); a `OneEvaluation` leaf is read -// once and held for both (1 read per run). +// Two roots reading one leaf: draining them one after another gives the +// same answers as polling them together. A leaf whose rows the store keeps +// (`None` or `AcrossEvaluations`) is read once per consumer (2 reads per +// run); a `OneEvaluation` leaf is read once and held for both (1 read per +// run). #[test] fn roots_reading_one_leaf_drain_in_order() { - for (leaf, reads_per_run) in [(Lifetime::None, 2), (ONE_EVALUATION, 1)] { + for (leaf, reads_per_run) in [ + (Lifetime::None, 2), + (Lifetime::AcrossEvaluations(StorageMedium::Memory), 2), + (ONE_EVALUATION, 1), + ] { let reads = within_timeout(move || { let mut dag = PhysicalDAG::default(); let starts = two_readers_of_one_leaf(&mut dag, leaf); @@ -376,8 +417,8 @@ fn roots_reading_one_leaf_drain_in_order() { } } -// One query whose two branches read one stored view (b17, g10), combined by -// an operator that reads its left branch to the end first. +// One root with two branches over one leaf, combined by an operator that +// reads its left branch to the end first. #[test] fn one_root_with_two_branches_over_one_leaf() { for (leaf, reads) in [(Lifetime::None, 2), (ONE_EVALUATION, 1)] { @@ -577,3 +618,82 @@ fn cancellation_releases_held_batches_of_unpolled_roots() { Some(Err(Error::Cancelled)) )); } + +/// Passes its input through and cancels the run from inside its own poll +/// once it has passed `after` batches, as an operator that hits a run-wide +/// stop condition would. +struct CancelsAfter { + after: usize, +} +impl PhysicalOperator for CancelsAfter { + fn name(&self) -> &str { + "CancelsAfter" + } + fn input_schemas(&self) -> Vec<()> { + vec![()] + } + fn output_schema(&self) {} + fn output_bytes(&self, _: &u64) -> usize { + 8 + } + fn start<'a>( + &'a self, + mut inputs: Vec>, + context: RunContext, + ) -> Result, Error> { + let mut passed = 0; + Ok(inputs + .remove(0) + .map(move |value| { + passed += 1; + if passed == self.after { + context.cancel(); + } + value.map(|value| *value) + }) + .boxed_local()) + } +} +/// A source, node 1 (`CancelsAfter` 3 batches, with lifetime `shared`) and +/// two `Identity` roots 2 and 3 over node 1. +fn cancelled_inside_a_shared_node(shared: Lifetime) -> PhysicalDAG<'static, u64, ()> { + let (mut source, _, _) = source(false); + source.end = MANY_BATCHES; + let mut dag = PhysicalDAG::default(); + dag.add(0, vec![], source, Lifetime::None).unwrap(); + dag.add(1, vec![0], CancelsAfter { after: 3 }, shared) + .unwrap(); + dag.add(2, vec![1], Identity, Lifetime::None).unwrap(); + dag.add(3, vec![1], Identity, Lifetime::None).unwrap(); + dag +} + +// An operator that cancels the run inside its own poll, while another +// consumer of its node is not being polled, ends every root with +// `Cancelled`: stopping the other producers does not drop their streams, +// whose inputs would borrow the producer being polled. +#[test] +fn cancel_inside_a_held_node_ends_every_root() { + let dag = cancelled_inside_a_shared_node(ONE_EVALUATION); + let run = query_context(1 << 20); + let outputs = dag.execute(&[2, 3], run.clone()).unwrap(); + let drained: Vec<_> = outputs + .into_iter() + .map(|output| values(block_on(output.collect::>()))) + .collect(); + assert_eq!( + drained, + vec![vec![Err(Error::Cancelled)], vec![Err(Error::Cancelled)]] + ); + assert_eq!(run.retained_bytes(), 0); +} + +// The same at ingestion time, over a streamed fan-out node. +#[test] +fn cancel_inside_an_ingestion_fan_out_ends_the_run() { + let dag = cancelled_inside_a_shared_node(Lifetime::None); + let run = ingestion_context(1 << 20); + let outputs = block_on(dag.execute_ingestion(&[2, 3], run.clone()).unwrap()); + assert!(matches!(outputs, Err(Error::Cancelled))); + assert_eq!(run.retained_bytes(), 0); +} diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 071eccef3..f3bc40193 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -14,7 +14,6 @@ use futures::{executor::block_on, StreamExt}; use planner_types::accuracy::AccuracyTarget; use planner_types::ir::schema::{DataType, *}; use planner_types::plan::physical_asap_dag::PhysicalASAPOperatorPayload; -use planner_types::stages::Lifetime; use planner_types::summary_type::*; use std::{collections::BTreeMap, sync::Arc}; @@ -90,13 +89,12 @@ fn input(samples: &[(&str, i64, f64)]) -> Batch { fn snapshot_plan(data: Batch) -> CompiledPhysicalDAG { CompiledPhysicalDAG::from_operators( BTreeMap::from([ - (0, (vec![], source(data), Lifetime::None)), + (0, (vec![], source(data))), ( 1, ( vec![0], Operator::current_series(schema(), 3, 0, 1, 60_000).unwrap(), - Lifetime::None, ), ), ]), @@ -198,17 +196,16 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { let plan = |data: Batch| { CompiledPhysicalDAG::from_operators( BTreeMap::from([ - (0, (vec![], source(data), Lifetime::None)), + (0, (vec![], source(data))), ( 1, ( vec![0], Operator::current_series(schema(), 3, 0, 1, 60_000).unwrap(), - Lifetime::None, ), ), - (2, (vec![1], build.clone(), Lifetime::None)), - (3, (vec![2], read.clone(), Lifetime::None)), + (2, (vec![1], build.clone())), + (3, (vec![2], read.clone())), ]), vec![3], ) diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index e8ecf4f50..bc889dd24 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -210,14 +210,7 @@ fn try_run( ) .unwrap(); operators.insert(300, (vec![merged], read)); - let compiled = CompiledPhysicalDAG::from_operators( - operators - .into_iter() - .map(|(id, (inputs, operator))| (id, (inputs, operator, Lifetime::None))) - .collect(), - vec![300], - ) - .unwrap(); + let compiled = CompiledPhysicalDAG::from_operators(operators, vec![300]).unwrap(); let dag = compiled .instantiate(Arc::new(InMemoryDataStore::default())) .unwrap(); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index c73b459c6..5dbdc04e1 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -1317,9 +1317,9 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { }; CompiledPhysicalDAG::from_operators( [ - (0, (vec![], source(left), Lifetime::None)), - (1, (vec![], source(right), Lifetime::None)), - (2, (vec![0, 1], join.clone(), Lifetime::None)), + (0, (vec![], source(left))), + (1, (vec![], source(right))), + (2, (vec![0, 1], join.clone())), ] .into(), vec![2], @@ -1439,14 +1439,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { }) .collect::>(); operators.insert(2, (vec![0, 1], binary.clone())); - let program = CompiledPhysicalDAG::from_operators( - operators - .into_iter() - .map(|(id, (inputs, operator))| (id, (inputs, operator, Lifetime::None))) - .collect(), - vec![2], - ) - .unwrap(); + let program = CompiledPhysicalDAG::from_operators(operators, vec![2]).unwrap(); let dag = program .instantiate(Arc::new(InMemoryDataStore::default())) .unwrap(); diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index e0f6dd706..0643bd71f 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -17,7 +17,6 @@ use planner_types::plan::physical_asap_dag::{ EdgeRole, GroupingEdgeCompatibility, PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPOperatorPayload, WindowEdgeCompatibility, }; -use planner_types::stages::Lifetime; use std::{collections::BTreeMap, sync::Arc, time::Duration}; fn schema() -> SchemaRef { @@ -96,9 +95,9 @@ fn over_sources(binary: Operator, left: Batch, right: Batch) -> Arc) -> Result 1 && computed[node.id] { + if consumers[node.id] > 1 { node.lifetime = Lifetime::OneEvaluation(StorageMedium::Memory); } } diff --git a/crates/types/src/test_support/forwarders.rs b/crates/types/src/test_support/forwarders.rs index 34213563b..e3092249c 100644 --- a/crates/types/src/test_support/forwarders.rs +++ b/crates/types/src/test_support/forwarders.rs @@ -33,8 +33,10 @@ pub fn matches_node(input: &CurrentSeriesInput, node: &Rc) -> bool } /// `physical_asap_dag::physical_asap_dag_at_query_time_with_node_ids`: `roots` -/// exported with every node at query time, not materialized and without θ, -/// with each exported node's operator node by node id. +/// exported as one DAG with every node at query time and without θ, with +/// each exported node's operator node by node id. A node with more than one +/// consumer, a leaf included, is `OneEvaluation`, as Stage 3 selects for a +/// shared query-time output; every other node is not materialized. pub fn physical_asap_dag_at_query_time_with_node_ids( roots: &[crate::ir::QueryRoot], ) -> Result< diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 25f9580fc..507a73930 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -517,10 +517,15 @@ An **input slot** is a position in a compiled DAG where data is read at run time, together with that position's schema, which is the DAG node's `output_schema`. There are two kinds: -| Slot | Where | Holds | Read by | -| --- | --- | --- | --- | -| **Raw data slot** | every `Scan` leaf | the `Scan`'s `Source`, schema, compiled predicates, and the window its consumers read (from the `TimeRange`, `TimeShift` and chain nodes above it) | `Source` | -| **Materialized view slot** | each view the DAG reads | the `MaterializedView`, its schema, and the rows a read selects: a chain's, the windows the `TimeRange`s above it select; any other view's, the window its state summarizes (the union of the windows of every `TimeRange` below it) | `MaterializedView` | +| Slot | Where | Holds | Lifetime | Read by | +| --- | --- | --- | --- | --- | +| **Raw data slot** | every `Scan` leaf | the `Scan`'s `Source`, schema, compiled predicates, and the window its consumers read (from the `TimeRange`, `TimeShift` and chain nodes above it) | the `Scan` node's | `Source` | +| **Materialized view slot** | each view the DAG reads | the `MaterializedView`, its schema, and the rows a read selects: a chain's, the windows the `TimeRange`s above it select; any other view's, the window its state summarizes (the union of the windows of every `TimeRange` below it) | the view node's, `AcrossEvaluations` | `MaterializedView` | + +The compiled DAG keeps each Planner node's `Lifetime` in one map, and the +runtime runs every node, slots included, by it: how a slot is read at +query time (by each consumer, or once and held) is the read rule in +[the executor README](../../crates/executor/README.md). ```rust pub fn compile(dag: &PhysicalASAPDAG, views: &[MaterializedView], roots: &[NodeId]) From 621a7b6649920617b17b712017251bb7476f2902 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:02:37 +0000 Subject: [PATCH 6/7] fix(executor): lifetimes only for compiled nodes; doc fixes from round 3 - Build CompiledPhysicalDAG::lifetimes from the nodes the roots reach. - from_operators doc: every node runs as Lifetime::None, so a shared computed node needs PhysicalDAG::add with a lifetime. - NodeRun, Producer, README: a leaf read per consumer runs once per consumer; constants hold their own rows; a held computed AcrossEvaluations node occurs only in a hand-built DAG. - cancel doc: operator state in a kept stream is freed when the caller drops the run's inputs. - views_at doc gives the validation reason; tutorial uses Scope::Ingestion / Scope::Query consistently. --- crates/executor/README.md | 26 +++++++++------- .../executor/src/physical_planner/compiled.rs | 11 ++++--- crates/executor/src/physical_planner/mod.rs | 4 +-- .../src/physical_planner/test_support.rs | 4 +-- crates/executor/src/runtime/mod.rs | 31 +++++++++++-------- .../src/tutorials/_05_executing_a_plan.md | 4 +-- 6 files changed, 46 insertions(+), 34 deletions(-) diff --git a/crates/executor/README.md b/crates/executor/README.md index 7c9c3a019..b9e8075e3 100644 --- a/crates/executor/README.md +++ b/crates/executor/README.md @@ -12,21 +12,25 @@ cancellation and byte accounting. Executions use the caller's worker and worker-local streams, with no internal thread pool. A node runs once per evaluation time in a run (an `@` anchor evaluates the -nodes below it at another time), as its `Lifetime` and the run's scope say: - -- Query time, a leaf whose rows the store keeps: `None` (a raw data scan the - plan does not materialize, or any other leaf such as a constant) or - `AcrossEvaluations` (a materialized view slot, or raw data kept at - ingestion). Each consumer reads it from the store on its own, and +nodes below it at another time; a leaf read per consumer runs once per +consumer), as its `Lifetime` and the run's scope say: + +- Query time, a leaf with `None` or `AcrossEvaluations` (rows a store or + the operator keeps): `None` (a raw data scan the plan does not + materialize, or any other leaf such as a constant, which holds its own + rows) or `AcrossEvaluations` (a materialized view slot, or raw data kept + at ingestion). Each consumer reads it on its own, and nothing is held. Stage 3 never gives a query-time `None` result more than one consumer; a view slot, or a leaf in a hand-built DAG, may have several. For example, an `AcrossEvaluations` scan read by two queries is two store reads. -- Query time, `OneEvaluation` (a computed node or a leaf), or a computed - `AcrossEvaluations` node, with more than one consumer: the node runs once, - to the end, before any consumer reads it, and every consumer reads its - finished output independently, as the planner charges one run whose - output is held. For example, a `OneEvaluation` scan read by three queries +- Query time, `OneEvaluation` (a computed node or a leaf) with more than + one consumer: the node runs once, to the end, before any consumer reads + it, and every consumer reads its finished output independently, as the + planner charges one run whose output is held. Likewise a computed + `AcrossEvaluations` node with more than one consumer, which only a + hand-built DAG has at query time (a compiled plan reads it through its + view slot). For example, a `OneEvaluation` scan read by three queries is one store read. The output is charged to the run's byte budget until every consumer has read it; over the budget the run fails with `MemoryLimit` (nothing spills). With one consumer the node streams to it, diff --git a/crates/executor/src/physical_planner/compiled.rs b/crates/executor/src/physical_planner/compiled.rs index 369c330e1..83ef73c0d 100644 --- a/crates/executor/src/physical_planner/compiled.rs +++ b/crates/executor/src/physical_planner/compiled.rs @@ -53,9 +53,9 @@ pub(super) struct RawDataSlot { pub struct CompiledPhysicalDAG { nodes: BTreeMap, roots: Vec, - /// The plan lifetime of each Planner node, which the runtime runs it by - /// (see `NodeRun` in the runtime). A helper node the lowering adds is - /// not in it and has `Lifetime::None`. Example: node 1, a stored view, + /// The plan lifetime of each compiled Planner node, which the runtime + /// runs it by (see `NodeRun` in the runtime). A helper node the lowering + /// adds is not in it and has `Lifetime::None`. Example: node 1, a stored view, /// maps to `AcrossEvaluations(Memory)`, and node 3, a query-time output /// two queries read, to `OneEvaluation(Memory)`. lifetimes: BTreeMap, @@ -63,7 +63,10 @@ pub struct CompiledPhysicalDAG { impl CompiledPhysicalDAG { /// Assemble already-lowered operators; leaves read no data store (for - /// example [`Operator::source`] holds its rows). + /// example [`Operator::source`] holds its rows). Every node runs as + /// `Lifetime::None`, so at query time a computed node may have at most + /// one consumer; for a shared one, build a [`PhysicalDAG`] with + /// `add(…, lifetime)`. pub fn from_operators( operators: BTreeMap, Operator)>, roots: Vec, diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 0b4809622..ffd9d1b6a 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -183,9 +183,9 @@ fn compile_internal( consumers: &consumers, roots: &compile_roots, }; - let lifetimes = nodes + let lifetimes = ordered .iter() - .map(|(&id, node)| (id, node.lifetime)) + .map(|&id| (id, nodes[&id].lifetime)) .collect(); let mut physical_dag = CompiledPhysicalDAG::new(roots.to_vec(), lifetimes); for id in ordered { diff --git a/crates/executor/src/physical_planner/test_support.rs b/crates/executor/src/physical_planner/test_support.rs index 945b50eb8..149794748 100644 --- a/crates/executor/src/physical_planner/test_support.rs +++ b/crates/executor/src/physical_planner/test_support.rs @@ -38,8 +38,8 @@ pub fn set_build_parameters<'a>( /// not a view is marked materialized at ingestion time, and it and /// every node below it run at ingestion time (their outputs and /// outgoing edges). A `OneEvaluation` node moved there becomes -/// `Lifetime::None`, as ingestion has no evaluation to hold it for: for -/// example a scan five tumbling windows share streams to all five. Every +/// `Lifetime::None`, as ingestion has no evaluation to hold it for (the +/// DAG's validation rejects `OneEvaluation` at ingestion): for example a scan five tumbling windows share streams to all five. Every /// view without a retention or coverage is /// retained for ever with a coverage of its own. For tests that bind /// states at those nodes, as a deployment binds its views. diff --git a/crates/executor/src/runtime/mod.rs b/crates/executor/src/runtime/mod.rs index ad51a540e..163c89a37 100644 --- a/crates/executor/src/runtime/mod.rs +++ b/crates/executor/src/runtime/mod.rs @@ -90,19 +90,21 @@ type ProducerKey = (NodeId, Option); /// it for callers. #[derive(Clone, Copy, PartialEq, Eq)] enum NodeRun { - /// Query time, a leaf whose rows the store keeps: `Lifetime::None` (a - /// raw data slot, or any other leaf such as a constant) or + /// Query time, a leaf with `Lifetime::None` (a raw data slot, or any + /// other leaf such as a constant, whose rows the operator keeps) or /// `AcrossEvaluations` (a materialized view slot, or raw data kept at - /// ingestion). Each consumer reads it on its own and nothing is held. + /// ingestion, whose rows a store keeps). Each consumer reads it on its own and nothing is held. /// Stage 3 never gives a query-time `None` result more than one /// consumer; a view slot, or a leaf in a hand-built DAG, may have /// several. Example: an `AcrossEvaluations` scan read by two queries is /// two store reads. ReadPerConsumer, - /// Query time, a `OneEvaluation` node (computed or a leaf), or a computed - /// `AcrossEvaluations` node, with more than one consumer: run to the end - /// once before any consumer reads it, then read by each consumer, as the - /// planner charges one run whose output is held. Example: a + /// Query time, a `OneEvaluation` node (computed or a leaf) with more + /// than one consumer: run to the end once before any consumer reads it, + /// then read by each consumer, as the planner charges one run whose + /// output is held. Likewise a computed `AcrossEvaluations` node with + /// more than one consumer, which only a hand-built DAG has at query time + /// (a compiled plan reads it through its view slot). Example: a /// `OneEvaluation` scan read by three queries is one store read whose /// batches every query reads. Held, @@ -270,17 +272,17 @@ fn start<'r, V: 'r, S: Clone + PartialEq + Debug + 'r>( /// batches the others hold. type RunProducers<'a, V> = Rc>>>>>; -/// One run of a node, read by every consumer it has in the run. +/// One run of a node, read by its consumers in the run (one, for a leaf +/// read per consumer). struct Producer<'a, V> { node: NodeId, operation: String, stream: Option>, size: Box usize + 'a>, context: RunContext, - /// At query time, a materialized node with several consumers - /// ([`NodeRun::Held`]) is computed to the end before any consumer reads - /// it, and its batches stay charged to the run until every consumer has - /// read them. + /// At query time, a [`NodeRun::Held`] node is computed to the end before + /// any consumer reads it, and its batches stay charged to the run until + /// every consumer has read them. hold_until_finished: bool, /// How many consumers subscribe to it in the run; its stream is dropped /// early only once all have subscribed and all have left. @@ -312,7 +314,10 @@ impl Producer<'_, V> { /// not yet read are released. The stream is kept, and dropped with the /// node's last input: the producer that saw the cancellation may be /// borrowed up the poll stack, and an input inside this stream, of - /// another consumer of it, would borrow it again on drop. + /// another consumer of it, would borrow it again on drop. So state an + /// operator keeps inside its stream (for example an accumulator's + /// reserved bytes) is freed only when the caller drops the run's + /// inputs, as callers do after `Cancelled`. fn cancel(&mut self) { if self.failure != Some(Error::Cancelled) { self.end(Some(Error::Cancelled)); diff --git a/crates/planner/src/tutorials/_05_executing_a_plan.md b/crates/planner/src/tutorials/_05_executing_a_plan.md index 168e380d9..c80d785b1 100644 --- a/crates/planner/src/tutorials/_05_executing_a_plan.md +++ b/crates/planner/src/tutorials/_05_executing_a_plan.md @@ -13,8 +13,8 @@ building a deployment runtime on top of the planner. and its roots into a `CompiledPhysicalDAG`, opening no reader. - Instantiate: `CompiledPhysicalDAG::instantiate` binds every raw data and materialized view slot to a `DataStore`, here an `InMemoryDataStore`. -- Execute: `PhysicalDAG::execute_ingestion` with a `RunContext` whose - `Scope` is `Ingestion` (to build views), or `PhysicalDAG::execute` with a +- Execute: `PhysicalDAG::execute_ingestion` with a `Scope::Ingestion` + context (to build views), or `PhysicalDAG::execute` with a `Scope::Query` context (to answer a query at an evaluation time). Each rejects the other's scope. - Reading the results: `execute_ingestion` returns every root's batches for From 88c83a059f25cdc6b9d0da2e4fe39cc9fd881b22 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 03:55:33 +0000 Subject: [PATCH 7/7] test(executor): give main's new executor tests a node Lifetime The grouped-limit, exact top-k and chain tests added on main build a PhysicalDAG with three-argument add calls; each node they add is read by one consumer, so it runs with Lifetime::None. --- crates/executor/tests/blocking_resources.rs | 10 ++++- crates/executor/tests/physical_semantics.rs | 46 ++++++++++++++++----- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index 3e0fc952b..b783608f9 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -158,8 +158,13 @@ fn sort_with_fetch_holds_groups_times_fetch_rows() { }) .collect(); let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input.clone(), batches).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), batches).unwrap(), + Lifetime::None, + ) + .unwrap(); let key = SortKey { column: 1, descending: true, @@ -169,6 +174,7 @@ fn sort_with_fetch_holds_groups_times_fetch_rows() { 1, vec![0], Operator::sort(input.clone(), vec![key], vec![0], fetch).unwrap(), + Lifetime::None, ) .unwrap(); let context = context(1024 * 1024); diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 8abc81110..f6958dc53 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -262,18 +262,25 @@ fn sort_with_fetch_over_ascending_input_equals_sort_then_limit() { .into_iter() .map(|rows| Batch::try_new(input.clone(), rows).unwrap()) .collect(); - dag.add(0, vec![], Operator::source(input.clone(), source).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), source).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 1, vec![0], Operator::sort(input.clone(), key.clone(), groups.clone(), None).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 2, vec![1], Operator::limit(input.clone(), fetch, 0, groups.clone()).unwrap(), + Lifetime::None, ) .unwrap(); assert_eq!( @@ -363,18 +370,25 @@ fn sort_with_fetch_equals_sort_then_limit() { .into_iter() .map(|rows| Batch::try_new(input.clone(), rows).unwrap()) .collect(); - dag.add(0, vec![], Operator::source(input.clone(), source).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), source).unwrap(), + Lifetime::None, + ) + .unwrap(); dag.add( 1, vec![0], Operator::sort(input.clone(), key.clone(), groups.clone(), None).unwrap(), + Lifetime::None, ) .unwrap(); dag.add( 2, vec![1], Operator::limit(input.clone(), fetch, 0, groups.clone()).unwrap(), + Lifetime::None, ) .unwrap(); let limited = collect(&dag, 2); @@ -396,11 +410,17 @@ fn try_chain( ops: Vec, ) -> Result>, String> { let mut dag = PhysicalDAG::default(); - dag.add(0, vec![], Operator::source(input.clone(), batches).unwrap()) - .unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), batches).unwrap(), + Lifetime::None, + ) + .unwrap(); let root = ops.len() as u64; for (index, op) in ops.into_iter().enumerate() { - dag.add(index as u64 + 1, vec![index as u64], op).unwrap(); + dag.add(index as u64 + 1, vec![index as u64], op, Lifetime::None) + .unwrap(); } block_on(async { let mut stream = dag @@ -591,13 +611,19 @@ fn identities_equal_but_for_an_excluded_label_aggregate_in_one_group() { .into_iter() .map(|rows| Batch::try_new(input.clone(), rows).unwrap()) .collect(); - dag.add(0, vec![], Operator::source(input.clone(), source).unwrap()) - .unwrap(); - dag.add(1, vec![0], project).unwrap(); + dag.add( + 0, + vec![], + Operator::source(input.clone(), source).unwrap(), + Lifetime::None, + ) + .unwrap(); + dag.add(1, vec![0], project, Lifetime::None).unwrap(); dag.add( 2, vec![1], Operator::aggregate(projected, vec![0], vec![("sum".into(), Reduction::Sum(1))]).unwrap(), + Lifetime::None, ) .unwrap(); let mut sums = collect(&dag, 2)