From 793642791a2249d161319a6ca4f9d60e029e59db Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:10:05 +0000 Subject: [PATCH 1/4] fix(executor): global aggregates follow the input's result kind, not the identity column A PromQL aggregate with no grouping keys over another aggregate's output (`count(sum by (job) (m >= 1000))`) answered one row over an empty input, and `sum`, `min`, `max`, `avg` and `quantile` over such an output declared nullable outputs that failed to install against the Planner's non-null schema. The executor chose SQL or PromQL rules by whether the input still had a series identity column, which an aggregate's output never has. The lowering now reads the IR's result kind of the rows a node reads (`input_rows_kind`: its input's, or for summary states the rows they were built from). Over a `Relation`, an aggregate, per-group build or merge into no groups opts into `with_emit_empty_global`, which also makes SUM, AVG, MIN, MAX and percentile measures nullable; over a PromQL vector none does, and a quantile is PromQL's. The identity-column test is gone from the aggregate's nullability, its run and the per-group build. --- .../src/operators/aggregate/columnar.rs | 40 +++-- .../executor/src/operators/aggregate/mod.rs | 37 ++-- crates/executor/src/operators/mod.rs | 78 +++++++++ crates/executor/src/operators/summary/mod.rs | 40 +---- crates/executor/src/operators/unchecked.rs | 41 +++-- crates/executor/src/physical_planner/mod.rs | 162 ++++++++++++------ crates/executor/tests/hydra_cms_execution.rs | 56 ++++-- .../tests/keyed_summary_row_weight.rs | 7 +- crates/executor/tests/physical_dag.rs | 24 ++- crates/executor/tests/physical_semantics.rs | 153 +++++++++++++++-- crates/executor/tests/promql_binary.rs | 7 +- crates/executor/tests/univmon_execution.rs | 6 +- crates/planner/tests/runtime_install.rs | 50 ++++++ crates/planner/tests/sql_table_scans.rs | 37 ++++ 14 files changed, 559 insertions(+), 179 deletions(-) diff --git a/crates/executor/src/operators/aggregate/columnar.rs b/crates/executor/src/operators/aggregate/columnar.rs index 8bf04a1e9..372319bcd 100644 --- a/crates/executor/src/operators/aggregate/columnar.rs +++ b/crates/executor/src/operators/aggregate/columnar.rs @@ -5,10 +5,11 @@ use super::{kahan_inc, percentile_cont, quantile, Reduction}; use crate::operators::common::{row_chunks, Workspace}; use crate::operators::group_table::GroupTable; +use crate::operators::{Kind, Operator}; use crate::runtime::{Cooperative, Input, RunContext}; use crate::values::arrow_columns::{self, ColumnDepth}; use crate::values::{Batch, SchemaRef, Value}; -use crate::{expressions::Expression, Error}; +use crate::Error; use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use arrow::array::{Array, ArrayRef, BooleanArray, Float64Array, Int64Array}; use futures::StreamExt; @@ -491,7 +492,6 @@ fn finish_measure( states: MeasureStates, measure: &Reduction, dtype: &DataType, - promql: bool, order: &[u32], ) -> Result { let values: Vec = match states { @@ -503,7 +503,7 @@ fn finish_measure( .iter() .map(|&g| { let (sum, c, seen) = sums[g as usize]; - if !seen && !promql { + if !seen { Value::Null } else if sum.is_infinite() { Value::Float64(sum) @@ -516,7 +516,7 @@ fn finish_measure( .iter() .map(|&g| { let average = averages[g as usize]; - if average.count == 0 && !promql { + if average.count == 0 { Value::Null } else { Value::Float64(average.value()) @@ -527,7 +527,7 @@ fn finish_measure( .iter() .map(|&g| { let (sum, count) = sums[g as usize]; - if count == 0 && !promql { + if count == 0 { Ok(Value::Null) } else if matches!(measure, Reduction::Avg(_)) { Ok(Value::Float64(sum as f64 / count as f64)) @@ -639,18 +639,25 @@ fn finish_measure( ) } -/// The exact `Aggregate`: the groups of `groups` in key order, each with -/// its `measures` (each over the rows its filter keeps). +/// The exact `Aggregate` `operator`: the groups of its `groups` in key +/// order, each with its `measures` (each over the rows its filter keeps). +/// With `emit_empty_global` and no groups, an empty input still has the +/// global group (SQL). pub(super) async fn aggregate( mut input: Input<'_, Batch>, - groups: &[usize], - measures: &[Reduction], - filters: &[Option], - input_schema: &SchemaRef, - output: &SchemaRef, + operator: &Operator, context: &RunContext, ) -> Result { - let promql = input_schema.has_promql_series_identity(); + let Kind::Aggregate { + groups, + measures, + filters, + emit_empty_global, + } = &operator.kind + else { + unreachable!("an aggregate's run") + }; + let (input_schema, output) = (&operator.inputs[0], &operator.output); // `now()` in a filter is this run's evaluation time. let filters: Vec> = filters .iter() @@ -715,8 +722,9 @@ pub(super) async fn aggregate( workspace.set_bytes(workspace_bytes + table.memory_bytes())?; } } - // PromQL sums over an empty vector emit no sample. - if table.len() == 0 && groups.is_empty() && !promql { + // A global SQL aggregate has one row over no input rows; a PromQL + // aggregate over an empty vector emits no sample. + if table.len() == 0 && groups.is_empty() && *emit_empty_global { table.add_empty_group()?; for state in &mut states { state.ensure_groups(1); @@ -732,7 +740,7 @@ pub(super) async fn aggregate( let FieldDataType::Plain(dtype) = &field.dtype else { return Err(invalid("aggregate measures are plain")); }; - arrays.push(finish_measure(state, measure, dtype, promql, &order)?); + arrays.push(finish_measure(state, measure, dtype, &order)?); } Batch::from_parts( output.clone(), diff --git a/crates/executor/src/operators/aggregate/mod.rs b/crates/executor/src/operators/aggregate/mod.rs index e4b578d36..6e3956f4b 100644 --- a/crates/executor/src/operators/aggregate/mod.rs +++ b/crates/executor/src/operators/aggregate/mod.rs @@ -35,8 +35,10 @@ impl Operator { } (DataType::Float64, false) } + // NULL for a group of NULLs, as SQL `SUM` (and for a + // global one over no row: `with_emit_empty_global`). Reduction::Sum(i) | Reduction::Avg(i) => { - let (t, _) = plain(&input, *i)?; + let (t, nullable) = plain(&input, *i)?; if !matches!(t, DataType::Int64 | DataType::Float64) { return Err(invalid("numeric aggregate input required")); } @@ -46,8 +48,7 @@ impl Operator { } else { t.clone() }, - !input.has_promql_series_identity() - && (groups.is_empty() || plain(&input, *i)?.1), + nullable, ) } Reduction::Quantile { column, .. } => { @@ -63,19 +64,16 @@ impl Operator { "a percentile requires Float64 input and a fraction in [0, 1]", )); } - // NULL for a group with no value, as SQL `SUM`: a global - // one over no row, or a group of NULLs. - ( - DataType::Float64, - groups.is_empty() || plain(&input, *column)?.1, - ) + // NULL for a group of NULLs, as SQL `SUM` (and for a + // global one over no row: `with_emit_empty_global`). + (DataType::Float64, plain(&input, *column)?.1) } Reduction::Min(i) | Reduction::Max(i) => { let (t, nullable) = plain(&input, *i)?; if !ordered(t) { return Err(invalid("ordered aggregate input required")); } - (t.clone(), nullable || groups.is_empty()) + (t.clone(), nullable) } Reduction::StdDev { column, population } | Reduction::Variance { column, population } => { @@ -113,6 +111,7 @@ impl Operator { groups, measures: measures.into_iter().map(|(_, r)| r).collect(), filters: vec![], + emit_empty_global: false, }, inputs: vec![input], output: schema(fields), @@ -128,6 +127,7 @@ impl Operator { filters: slot, measures, groups, + .. } = &mut self.kind else { return Err(invalid("measure filters need an aggregate")); @@ -333,22 +333,7 @@ pub(super) fn execute<'a>( results.len(), ) } - Kind::Aggregate { - groups, - measures, - filters, - } => { - columnar::aggregate( - input, - groups, - measures, - filters, - &operator.inputs[0], - &output, - &context, - ) - .await - } + Kind::Aggregate { .. } => columnar::aggregate(input, operator, &context).await, _ => unreachable!(), } }) diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index ded0132f9..f9417a739 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -156,6 +156,12 @@ enum Kind { /// Per-measure row filters (SQL `FILTER (WHERE …)`); empty when none. #[serde(default, skip_serializing_if = "Vec::is_empty")] filters: Vec>, + /// Whether an empty input still has the global group, as a SQL + /// `SELECT count(*) FROM t WHERE false` has one row (0); a PromQL + /// `count(v)` over an empty vector has no sample. Only an aggregate + /// with no groups; `false` (no output row) by default. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + emit_empty_global: bool, }, SemiJoin { keys: Vec<(usize, usize)>, @@ -179,6 +185,11 @@ enum Kind { /// Rows for which this is not true update no state; their group is kept. #[serde(default, skip_serializing_if = "Option::is_none")] filter: Option>, + /// Whether an empty input still has the global group, with an empty + /// state, as a global SQL aggregate has one row over no input rows. + /// Only a build with no groups; `false` (no output row) by default. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + emit_empty_global: bool, }, /// One keyed state per group whose items take a row weight, e.g. a CMS /// with heap (built by [`Operator::per_group_keyed_summary_build`]). @@ -352,6 +363,73 @@ impl Operator { pub fn schema(&self) -> SchemaRef { self.output.clone() } + + /// Give an aggregate with no groups SQL's rule for an empty input: its + /// one global group still has a row, as `SELECT count(*) FROM t WHERE + /// false` answers 0 and `SELECT sum(x) FROM t WHERE false` NULL. Without + /// it (the default) an empty input has no row, as a PromQL `count(v)` + /// over an empty vector has no sample. + /// + /// An `Aggregate`'s SUM, AVG, MIN, MAX and percentile measures become + /// nullable (NULL over no row). A `PerGroupSummaryBuild` outputs an empty + /// state of its own θ, and a `SummaryMerge` one of `parameters`: the θ + /// of the states it merges (the exported build node's `parameters`), + /// `None` for an exact family and for the other two. + pub fn with_emit_empty_global( + mut self, + parameters: Option, + ) -> Result { + match &mut self.kind { + Kind::Aggregate { + groups, + measures, + emit_empty_global, + .. + } if groups.is_empty() && parameters.is_none() => { + let mut output = (*self.output).clone(); + for (field, measure) in output.fields.iter_mut().zip(measures.iter()) { + if matches!( + measure, + Reduction::Sum(_) + | Reduction::Avg(_) + | Reduction::Min(_) + | Reduction::Max(_) + | Reduction::Percentile { .. } + ) { + field.nullable = true; + } + } + *emit_empty_global = true; + self.output = Arc::new(output); + } + Kind::PerGroupSummaryBuild { + groups, + emit_empty_global, + .. + } if groups.is_empty() && parameters.is_none() => *emit_empty_global = true, + Kind::SummaryMerge { + state, + groups, + emit_empty_global, + parameters: slot, + } if groups.is_empty() => { + // An empty state of this family and θ can be built. + crate::StateColumnUpdater::new( + &self.inputs[0].fields[*state].dtype, + parameters.as_ref(), + )?; + *emit_empty_global = true; + *slot = parameters; + } + _ => { + return Err(invalid( + "an empty global row needs an aggregate, build or merge into no groups \ + (only a merge takes the θ of its empty state)", + )) + } + } + Ok(self) + } } impl PhysicalOperator for Operator { fn requires_bounded_input(&self) -> bool { diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 7faab29cd..7e0e95cd6 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -277,6 +277,7 @@ impl Operator { time, groups, filter: None, + emit_empty_global: false, }, inputs: vec![input], output: schema(fields), @@ -308,36 +309,6 @@ impl Operator { output: schema(fields), }) } - /// A merge into no groups that outputs one empty state when it has no - /// input state: the global SQL aggregate a rollup of a relation's - /// grouped states to no keys stands for, whose one row is NULL over no - /// input rows. `parameters` is the merged states' θ, which the empty - /// state takes: the exported build node's `parameters` (`None` for an - /// exact family). - pub fn with_emit_empty_global( - mut self, - parameters: Option, - ) -> Result { - match &mut self.kind { - Kind::SummaryMerge { - state, - groups, - emit_empty_global, - parameters: slot, - } if groups.is_empty() => { - // An empty state of this family and θ can be built. - StateColumnUpdater::new(&self.inputs[0].fields[*state].dtype, parameters.as_ref())?; - *emit_empty_global = true; - *slot = parameters; - } - _ => { - return Err(invalid( - "an empty global state needs a merge into no groups", - )) - } - } - Ok(self) - } pub fn evaluation( input: SchemaRef, state: usize, @@ -459,6 +430,7 @@ pub(super) fn execute<'a>( time, groups, filter, + emit_empty_global, } => Ok(futures::stream::once(async move { build_per_group_summary( input, @@ -469,7 +441,7 @@ pub(super) fn execute<'a>( time: *time, groups, filter: filter.as_deref(), - emit_empty_global: !input_schema.has_promql_series_identity(), + emit_empty_global: *emit_empty_global, }, &input_schema, &output, @@ -613,7 +585,8 @@ struct PerGroupBuild<'a> { time: Option, groups: &'a [usize], filter: Option<&'a Expression>, - /// Whether an empty input still has the global group (SQL; not PromQL). + /// Whether an empty input still has the global group (SQL; not PromQL): + /// the kind's `emit_empty_global`. emit_empty_global: bool, } @@ -634,7 +607,8 @@ async fn build_per_group_summary( let mut updater = StateColumnUpdater::new(build.family, build.parameters)?; let mut work = Cooperative::new(context); let mut workspace = Workspace::new(context)?; - // PromQL aggregation of an empty vector produces no sample. + // A global SQL aggregate has one row over no input rows; a PromQL + // aggregation of an empty vector produces no sample. if build.groups.is_empty() && build.emit_empty_global { table.add_empty_group()?; updater.add_group()?; diff --git a/crates/executor/src/operators/unchecked.rs b/crates/executor/src/operators/unchecked.rs index 69c47e522..21796d5f5 100644 --- a/crates/executor/src/operators/unchecked.rs +++ b/crates/executor/src/operators/unchecked.rs @@ -152,13 +152,20 @@ impl TryFrom for Operator { groups, measures, filters, + emit_empty_global, } => { if groups.len() + measures.len() != output.fields.len() { return Err(invalid("aggregate width mismatch")); } let names = output.fields[groups.len()..].iter().map(|f| f.name.clone()); - Operator::aggregate(input(0)?, groups, names.zip(measures).collect())? - .with_measure_filters(filters)? + let aggregate = + Operator::aggregate(input(0)?, groups, names.zip(measures).collect())? + .with_measure_filters(filters)?; + if emit_empty_global { + aggregate.with_emit_empty_global(None)? + } else { + aggregate + } } Kind::SemiJoin { keys, @@ -190,17 +197,25 @@ impl TryFrom for Operator { time, groups, filter, - } => with_filter( - Operator::per_group_summary_build( - input(0)?, - family, - parameters, - value, - time, - groups, - )?, - filter, - )?, + emit_empty_global, + } => { + let build = with_filter( + Operator::per_group_summary_build( + input(0)?, + family, + parameters, + value, + time, + groups, + )?, + filter, + )?; + if emit_empty_global { + build.with_emit_empty_global(None)? + } else { + build + } + } Kind::PerGroupKeyedSummaryBuild { family, parameters, diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index f15c12e96..ebecd8c0f 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -28,7 +28,9 @@ use planner_types::summary_type::{ // `promql_rows` reads `OperatorNode` through `super::*` only in its // test-support helpers. #[cfg_attr(not(any(test, feature = "test-support")), allow(unused_imports))] -use planner_types::ir::{ASAPOp, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind}; +use planner_types::ir::{ + ASAPOp, NonASAPOp, OperatorNode, OperatorResultKind, ScalarExpr, TimeRangeKind, +}; mod logical; use std::{ collections::{BTreeMap, BTreeSet}, @@ -256,6 +258,8 @@ fn compile_internal( .iter() .map(|id| Arc::new(nodes[id].output_schema.clone())) .collect::>(); + let input_rows = input_rows_kind(&restored[&id]); + let over_relation = input_rows == OperatorResultKind::Relation; if matches!(node.payload, Payload::ASAP(ASAPOp::SummaryMerge { .. })) && inputs.len() > 1 { @@ -663,6 +667,7 @@ fn compile_internal( update, reduction, window_end, + over_relation, )?, }; let build = match filter { @@ -812,7 +817,7 @@ fn compile_internal( } if let Payload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) = &node.payload { // Exact counts evaluate to Int64; PromQL declares a Float64 sample. - let evaluation = bind_operation(node, &schemas) + let evaluation = bind_operation(node, &schemas, over_relation) .map_err(|error| invalid(format!("node {id}: {error}")))?; let actual = evaluation.schema(); let converted = actual.fields.iter().zip(&output.fields).position(|(a, d)| { @@ -861,7 +866,7 @@ fn compile_internal( // state covers this evaluation's window, so merge by // series identity and attach the run scope's timestamp, // as a per-series `SummaryAgg` does. - let merged = bind_operation(node, &schemas)?; + let merged = bind_operation(node, &schemas, over_relation)?; let compact = merged.schema(); physical_dag.add(helper_id(id, 1)?, inputs, merged)?; physical_dag.add( @@ -872,23 +877,6 @@ fn compile_internal( continue; } } - // A rollup of a relation's states to no keys is a global SQL - // aggregate: one row, an empty state, over no input rows. - if let Payload::ASAP(ASAPOp::SummaryMerge { - group_by: Some(keys), - .. - }) = &node.payload - { - if keys.is_empty() && built_from_relation(&restored[&id]) { - // The merge carries the θ of the states it reads. - let parameters = node.parameters.clone(); - let operator = compile_node(node, &schemas) - .and_then(|merge| merge.with_emit_empty_global(parameters)) - .map_err(|error| invalid(format!("node {id}: {error}")))?; - physical_dag.add(id, inputs, operator)?; - continue; - } - } // A window over a table or a time series (and a tumbling window's // shift under it) runs natively on the time column. if let Some(operator) = time_window_operator(id, &graph, &schemas) @@ -897,7 +885,7 @@ fn compile_internal( physical_dag.add(id, inputs, operator.with_output_schema(output)?)?; continue; } - let mut operator = compile_node(node, &schemas) + let mut operator = compile_node(node, &schemas, input_rows) .map_err(|error| invalid(format!("node {id}: {error}")))?; if operator.is_counter_evaluation() { if let Some((offset, lookback)) = counter_window(id, &nodes, &dependencies)? { @@ -916,18 +904,37 @@ fn compile_internal( Ok(physical_dag) } -/// Whether every summary state `state` merges was built from a relation's -/// rows (a SQL table's), where a global aggregate has a row over no input -/// row; not from a PromQL vector's, where it has none. -fn built_from_relation(state: &std::rc::Rc) -> bool { - match state.asap() { - Some(ASAPOp::SummaryMerge { children, .. }) => children.iter().all(built_from_relation), +/// The result kind of the rows `node` reads: its (first) input's, or, when +/// that input is summary states, the kind of the rows they were built from. +/// It decides an aggregate's rule for an empty input: a global aggregate +/// over a SQL table's `Relation` has one row over no input rows +/// (`SELECT count(*) FROM t WHERE false` is 0), over a PromQL vector it has +/// none (`count(sum by (job) (m >= 1000))` is empty when no value reaches +/// 1000), whether or not its input still has a series identity column. +fn input_rows_kind(node: &OperatorNode) -> OperatorResultKind { + node.operator + .children() + .first() + .map_or(node.result_kind, |input| rows_kind(input)) +} + +/// The result kind of `node`'s rows, or, for summary states, of the rows +/// they were built from (`State` when that is not known). +fn rows_kind(node: &std::rc::Rc) -> OperatorResultKind { + if node.result_kind != OperatorResultKind::State { + return node.result_kind; + } + match node.asap() { Some( ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => child.result_kind == planner_types::ir::OperatorResultKind::Relation, + ) => rows_kind(child), + // The states of one memo group are built from rows of one kind. + Some(ASAPOp::SummaryMerge { children, .. }) => { + children.first().map_or(node.result_kind, rows_kind) + } // A filter keeping whole states reads them from their build. - _ => planner_types::ir::operator::asap::selected_from(state) - .is_some_and(|states| built_from_relation(&states)), + _ => planner_types::ir::operator::asap::selected_from(node) + .map_or(node.result_kind, |states| rows_kind(&states)), } } @@ -973,14 +980,34 @@ pub(crate) fn require_sound_merge(input: &SchemaRef) -> Result<(), Error> { /// that lowers to one operator. A node that lowers to several (an exact /// top-k, `count_values`, or a `without` grouping over a series identity) /// is lowered only by [`compile`]; here it is refused. -pub fn compile_node(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { +/// `input_rows_kind` is the result kind of the rows the node reads (for +/// summary states, of the rows they were built from): an aggregate, build +/// or merge into no groups over a `Relation` has SQL's one row over no +/// input rows, and its quantile is SQL's percentile; over a PromQL vector +/// it has no row and PromQL's quantile. +pub fn compile_node( + node: &PhysicalASAPDAGNode, + inputs: &[SchemaRef], + input_rows_kind: OperatorResultKind, +) -> Result { for schema in inputs { crate::values::validate_schema(schema)?; } - bind_operation(node, inputs)?.with_output_schema(Arc::new(node.output_schema.clone())) + bind_operation( + node, + inputs, + input_rows_kind == OperatorResultKind::Relation, + )? + .with_output_schema(Arc::new(node.output_schema.clone())) } -fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { +/// `over_relation`: whether the node reads a SQL relation's rows (or +/// states built from them), not a PromQL vector's. +fn bind_operation( + node: &PhysicalASAPDAGNode, + inputs: &[SchemaRef], + over_relation: bool, +) -> Result { if let Payload::NonASAP(NonASAPOp::BinaryOp { operator, return_bool, @@ -1259,7 +1286,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { Reduction::Quantile { @@ -1270,9 +1297,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result - { + AggIntent::Quantile { col, q, .. } if over_relation => { Reduction::Percentile { column: column(*col)?, q: *q, @@ -1287,13 +1312,20 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result>()?; - Operator::aggregate(input.clone(), groups(input, keys)?, measures)? + let groups = groups(input, keys)?; + let global = groups.is_empty(); + let aggregate = Operator::aggregate(input.clone(), groups, measures)? .with_measure_filters( filters .iter() .map(|f| f.as_ref().map(|f| expression(&f.0, input)).transpose()) .collect::>()?, - ) + )?; + if global && over_relation { + aggregate.with_emit_empty_global(None) + } else { + Ok(aggregate) + } } _ => Err(invalid("value operation has no native implementation")), }, @@ -1317,6 +1349,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result build.with_row_filter(expression(&filter.0, input)?), @@ -1343,7 +1376,16 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { if let SketchStatistic::TopK { k } = query { @@ -1381,7 +1423,9 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result, + over_relation: bool, ) -> Result { let groups = |input: &SchemaRef, keys: &GroupKeys| -> Result, Error> { let mut columns = groups(input, keys)?; columns.extend(window_end); Ok(columns) }; + // A state per group of `groups` over `value`. + let per_group = |value: usize, groups: Vec| -> Result { + let global = groups.is_empty(); + let build = Operator::per_group_summary_build( + input.clone(), + family.clone(), + parameters.cloned(), + value, + input.time_index, + groups, + )?; + if global && over_relation { + build.with_emit_empty_global(None) + } else { + Ok(build) + } + }; if matches!( family, FieldDataType::Sketch(planner_types::summary_type::SketchAlgorithm::HydraCms) @@ -1431,14 +1493,7 @@ fn summary_agg_build( "summary construction requires explicit grouping columns", )); }; - return Operator::per_group_summary_build( - input.clone(), - family.clone(), - parameters.cloned(), - named_column(input, item)?, - input.time_index, - groups(input, keys)?, - ); + return per_group(named_column(input, item)?, groups(input, keys)?); } if let Some(item) = &update.item { let PlannerReduction::Reduce(keys) = reduction else { @@ -1483,14 +1538,7 @@ fn summary_agg_build( "summary construction requires explicit grouping columns", )); }; - Operator::per_group_summary_build( - input.clone(), - family.clone(), - parameters.cloned(), - named_column(input, column)?, - input.time_index, - groups(input, keys)?, - ) + per_group(named_column(input, column)?, groups(input, keys)?) } /// What each row adds in a keyed build: a projected weight column, or a /// positive integer constant (1 for a row count) read without a column. diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index bc889dd24..32b690814 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -168,7 +168,12 @@ fn try_run( batches: Vec, ) -> Result, String> { let input = input_schema(); - let build = compile_node(&summary_agg(update, filter), std::slice::from_ref(&input)).unwrap(); + let build = compile_node( + &summary_agg(update, filter), + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap(); let state = build.schema(); let mut operators = BTreeMap::new(); let mut states = vec![]; @@ -199,6 +204,7 @@ fn try_run( (*state).clone(), ), std::slice::from_ref(&state), + planner_types::ir::OperatorResultKind::Relation, ) .unwrap(); operators.insert(201, (vec![200], merge)); @@ -207,6 +213,7 @@ fn try_run( let read = compile_node( &estimate(3, point("checkout"), DataType::Float64), std::slice::from_ref(&state), + planner_types::ir::OperatorResultKind::Relation, ) .unwrap(); operators.insert(300, (vec![merged], read)); @@ -327,9 +334,13 @@ fn positive_constant_weight_adds_the_constant_per_row() { assert_eq!(doubled[job], 2.0 * unit[job], "{job}"); } for weight in [0.0, -2.0, 2.5, f64::INFINITY] { - let error = compile_node(&summary_agg(constant(weight), None), &[input_schema()]) - .err() - .unwrap_or_else(|| panic!("constant weight {weight} accepted")); + let error = compile_node( + &summary_agg(constant(weight), None), + &[input_schema()], + planner_types::ir::OperatorResultKind::Relation, + ) + .err() + .unwrap_or_else(|| panic!("constant weight {weight} accepted")); assert!(error.to_string().contains("HydraCms"), "{weight}: {error}"); } } @@ -349,7 +360,12 @@ fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { (constant, serde_json::json!({"Constant": 2})), (column, serde_json::json!({"Column": 2})), ] { - let build = compile_node(&summary_agg(update, None), &[input_schema()]).unwrap(); + let build = compile_node( + &summary_agg(update, None), + &[input_schema()], + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap(); let wire = serde_json::to_value(&build).unwrap(); let fields = &wire["kind"]["AllGroupsSummaryBuild"]; assert_eq!(fields["row_weight"], expected, "{wire}"); @@ -399,9 +415,13 @@ fn constant_weight_above_i32_is_refused_at_validation() { error.contains("HydraCms") && error.contains("i32::MAX"), "{error}" ); - let error = compile_node(&summary_agg(update, None), &[input_schema()]) - .err() - .expect("constant above i32::MAX compiled"); + let error = compile_node( + &summary_agg(update, None), + &[input_schema()], + planner_types::ir::OperatorResultKind::Relation, + ) + .err() + .expect("constant above i32::MAX compiled"); assert!( error.to_string().contains("HydraCms") && error.to_string().contains("i32::MAX"), "{error}" @@ -431,16 +451,24 @@ fn invalid_weight_values_fail_the_build() { // rather than computed from sketchlib internals. #[test] fn group_total_is_rejected_until_sketchlib_offers_it() { - let state = compile_node(&summary_agg(unit_count(), None), &[input_schema()]) - .unwrap() - .schema(); + let state = compile_node( + &summary_agg(unit_count(), None), + &[input_schema()], + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap() + .schema(); let total = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, }; - let error = compile_node(&estimate(3, total, DataType::Int64), &[state]) - .err() - .expect("group total"); + let error = compile_node( + &estimate(3, total, DataType::Int64), + &[state], + planner_types::ir::OperatorResultKind::Relation, + ) + .err() + .expect("group total"); assert!( error .to_string() diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index fe1873d52..cd654507b 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -175,7 +175,12 @@ fn query_time_count_heap_binds_a_constant_row_weight() { selected_physical_candidate: None, view_coverage: None, }; - let build = compile_node(&node, std::slice::from_ref(&input)).unwrap(); + let build = compile_node( + &node, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap(); let wire = serde_json::to_value(&build).unwrap(); assert_eq!( wire["kind"]["PerGroupKeyedSummaryBuild"]["row_weight"], diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 5dbdc04e1..15d55d361 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -305,7 +305,10 @@ fn exact_integer_and_empty_extrema() { .add( 1, vec![0], - Operator::aggregate(schema, vec![], vec![("min".into(), Reduction::Min(0))]).unwrap(), + Operator::aggregate(schema, vec![], vec![("min".into(), Reduction::Min(0))]) + .unwrap() + .with_emit_empty_global(None) + .unwrap(), Lifetime::None, ) .unwrap(); @@ -646,7 +649,8 @@ fn bind_physical_asap_dag_before_execution() { assert!(bind(&scanned, data(), &[1]).is_err()); } -// A completed empty population has an exact zero count, with integer output. +// A completed empty population of a global SQL count has an exact zero +// count, with integer output. #[test] fn empty_exact_count_is_an_integer_state_evaluation() { let input = schema(&[("value", DataType::Float64, false)]); @@ -658,6 +662,8 @@ fn empty_exact_count_is_an_integer_state_evaluation() { None, vec![], ) + .unwrap() + .with_emit_empty_global(None) .unwrap(); let read = Operator::evaluation( build.schema(), @@ -1301,7 +1307,12 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { let join = if certified { Operator::certified_semi_join(schema.clone(), schema.clone(), vec![(0, 0)]).unwrap() } else { - compile_node(&node, &[schema.clone(), schema.clone()]).unwrap() + compile_node( + &node, + &[schema.clone(), schema.clone()], + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap() }; // Each side is a leaf holding its keys. let plan = |left: Vec<&str>, right: Vec<&str>| { @@ -1405,7 +1416,12 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { selected_physical_candidate: None, view_coverage: None, }; - let binary = compile_node(&node, &[input.clone(), input.clone()]).unwrap(); + let binary = compile_node( + &node, + &[input.clone(), input.clone()], + planner_types::ir::OperatorResultKind::InstantVector, + ) + .unwrap(); for (right, expected) in [ (vec![("b", 2, 3.), ("a", 1, 2.)], Some(vec![8., 17.])), (vec![("b", 2, 3.)], None), diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index f6958dc53..68fae56cf 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -729,7 +729,8 @@ fn grouping_canonicalizes_null_nan_and_signed_zero() { assert!(result.iter().all(|r| matches!(r[1], Value::Int64(2)))); } -// Global empty input yields one aggregate row; grouped empty input yields none. +// Global empty input yields one aggregate row under SQL's rule +// (`with_emit_empty_global`); grouped empty input yields none. #[test] fn aggregate_empty_and_all_null_follow_asap_contract() { let input = schema(&[("v", DataType::Int64, true)]); @@ -748,6 +749,8 @@ fn aggregate_empty_and_all_null_follow_asap_contract() { ("max".into(), Reduction::Max(0)), ], ) + .unwrap() + .with_emit_empty_global(None) .unwrap(); let result = unary(input.clone(), batches, op); assert_eq!(result.len(), 1); @@ -823,8 +826,12 @@ fn global_extrema_bind_with_planner_derived_schema() { selected_physical_candidate: None, view_coverage: None, }; - let operator = compile_node(&node, std::slice::from_ref(&input)) - .expect("global extremum should bind to its Planner schema"); + let operator = compile_node( + &node, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) + .expect("global extremum should bind to its Planner schema"); assert!(operator.schema().fields[0].nullable); let empty = unary(input.clone(), vec![], operator.clone()); assert!(matches!(empty[0][0], Value::Null)); @@ -1301,7 +1308,8 @@ fn zero_column_output_obeys_memory_limit() { assert_eq!(run.retained_bytes(), 0); } -// Empty exact-state finalization must preserve ordinary global MIN/MAX null semantics. +// Empty exact-state finalization must preserve ordinary global SQL MIN/MAX +// null semantics. #[test] fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { use planner_types::summary_type::{ExactKind, ExactParams}; @@ -1319,6 +1327,8 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { None, vec![], ) + .unwrap() + .with_emit_empty_global(None) .unwrap(); let state = build.schema(); let mut dag = PhysicalDAG::default(); @@ -1465,7 +1475,14 @@ fn measure_filters_keep_groups_without_matching_rows() { ("all", plain(DataType::Int64), false), ], ); - let operator = round_trip(compile_node(&node, std::slice::from_ref(&input)).unwrap()); + let operator = round_trip( + compile_node( + &node, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap(), + ); assert_eq!( printed(&unary(input, vec![rows], operator)), ["a 1 1.0 2", "b 0 NULL 1", "c 1 4.0 1"] @@ -1493,7 +1510,12 @@ fn filtered_sum_declared_non_null_is_rejected() { ("s", FieldDataType::Plain(DataType::Float64), false), ], ); - assert!(compile_node(&node, &[input]).is_err()); + assert!(compile_node( + &node, + &[input], + planner_types::ir::OperatorResultKind::Relation + ) + .is_err()); } /// A filtered `SummaryAgg` updates its state only from kept rows but keeps @@ -1544,7 +1566,14 @@ fn filtered_summary_build_keeps_empty_groups() { ], ); asap_executor::physical_planner::test_support::set_build_parameters([&mut build], None); - let build = round_trip(compile_node(&build, std::slice::from_ref(&input)).unwrap()); + let build = round_trip( + compile_node( + &build, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap(), + ); let read = planner_node( PhysicalASAPOperatorPayload::ASAP(read), &[ @@ -1552,7 +1581,12 @@ fn filtered_summary_build_keeps_empty_groups() { ("value", FieldDataType::Plain(result), nullable), ], ); - let read = compile_node(&read, &[build.schema()]).unwrap(); + let read = compile_node( + &read, + &[build.schema()], + planner_types::ir::OperatorResultKind::Relation, + ) + .unwrap(); let mut dag = PhysicalDAG::default(); dag.add( 0, @@ -1598,7 +1632,11 @@ fn filtered_summary_build_is_rejected_at_ingestion_time() { ], ); node.output_state = ExecutionDataState::INGESTION_SUMMARY; - compile_node(&node, std::slice::from_ref(&input)) + compile_node( + &node, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) }; assert!(build(None).is_ok()); let error = build(keep()).err().expect("filtered ingestion-time build"); @@ -1665,6 +1703,87 @@ fn exact_aggregate_node( } } +// A global aggregate follows the result kind of the rows it reads, not +// whether they carry a series identity column. Over `[job, v]`, the output +// of `max by (job) (m)` (no identity column), PromQL's `sum(…)`, `avg`, +// `min`, `max` and `quantile(0.5, …)` are non-null and answer no sample over +// an empty vector; over the same columns as a SQL relation they are +// nullable and answer one NULL row over no row. Over jobs a (1) and b (3) +// both answer 4, 2, 1, 3 and 2. +#[test] +fn a_global_aggregate_follows_its_input_rows_kind() { + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::AggIntent; + use planner_types::ir::OperatorResultKind; + + let input = schema(&[ + ("job", DataType::Utf8, false), + ("v", DataType::Float64, false), + ]); + let rows = vec![ + vec![Value::Utf8("a".into()), Value::Float64(1.0)], + vec![Value::Utf8("b".into()), Value::Float64(3.0)], + ]; + for (measure, expected) in [ + (AggIntent::Sum { col: Some(1) }, 4.0), + (AggIntent::Avg { col: Some(1) }, 2.0), + (AggIntent::Min { col: Some(1) }, 1.0), + (AggIntent::Max { col: Some(1) }, 3.0), + ( + AggIntent::Quantile { + col: Some(1), + q: 0.5, + }, + 2.0, + ), + ] { + let mut node = exact_aggregate_node(measure.clone(), "value", DataType::Float64); + // A PromQL vector: non-null, no sample over an empty vector. + let promql = compile_node( + &node, + std::slice::from_ref(&input), + OperatorResultKind::InstantVector, + ) + .unwrap_or_else(|error| panic!("{measure:?}: {error}")); + assert!( + unary(input.clone(), vec![], promql.clone()).is_empty(), + "{measure:?}" + ); + let answer = unary(input.clone(), vec![rows.clone()], promql); + assert!( + matches!(answer.as_slice(), [row] if matches!(row[0], Value::Float64(v) if v == expected)), + "{measure:?}: {answer:?}" + ); + // A SQL relation: NULL over no row, so a non-null output is refused. + assert!(compile_node( + &node, + std::slice::from_ref(&input), + OperatorResultKind::Relation + ) + .is_err()); + node.output_schema.fields[0].nullable = true; + // The rule survives serialization. + let sql = round_trip( + compile_node( + &node, + std::slice::from_ref(&input), + OperatorResultKind::Relation, + ) + .unwrap_or_else(|error| panic!("{measure:?}: {error}")), + ); + let empty = unary(input.clone(), vec![], sql.clone()); + assert!( + matches!(empty.as_slice(), [row] if matches!(row[0], Value::Null)), + "{measure:?}: {empty:?}" + ); + let answer = unary(input.clone(), vec![rows.clone()], sql); + assert!( + matches!(answer.as_slice(), [row] if matches!(row[0], Value::Float64(v) if v == expected)), + "{measure:?}: {answer:?}" + ); + } +} + // Exact frequency intents bind to native reducers over typed keys, without a // sketch or numeric key conversion; empty and all-NULL input give 0. #[test] @@ -1704,8 +1823,12 @@ fn exact_frequency_intents_execute_typed_keys_and_empty_input() { ), ] { let node = exact_aggregate_node(measure, name, DataType::Float64); - let operator = compile_node(&node, std::slice::from_ref(&input)) - .expect("exact frequency intent binds"); + let operator = compile_node( + &node, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) + .expect("exact frequency intent binds"); let rows = values .iter() .flat_map(|v| [vec![v.clone()], vec![v.clone()]]) @@ -1809,8 +1932,12 @@ fn exact_cardinality_binds_and_executes_typed_tuples() { for (cols, expected) in [(vec![0], 2), (vec![0, 1], 3)] { let node = exact_aggregate_node(AggIntent::Cardinality { cols }, "distinct", DataType::Int64); - let operator = - compile_node(&node, std::slice::from_ref(&input)).expect("exact distinct intent binds"); + let operator = compile_node( + &node, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + ) + .expect("exact distinct intent binds"); let rows = vec![ vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 0643bd71f..03efa35a9 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -86,7 +86,12 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { selected_physical_candidate: None, view_coverage: None, }; - compile_node(&node, &[schema.clone(), schema]).unwrap() + compile_node( + &node, + &[schema.clone(), schema], + planner_types::ir::OperatorResultKind::InstantVector, + ) + .unwrap() } /// `binary` (node 2) over the sources `left` (node 0) and `right` (node 1), /// bound to a store with no data: the sources hold their rows. diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 1866ac810..7c1610126 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -199,7 +199,11 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() selected_physical_candidate: None, view_coverage: None, }; - let build = compile_node(&node, std::slice::from_ref(&input))?; + let build = compile_node( + &node, + std::slice::from_ref(&input), + planner_types::ir::OperatorResultKind::Relation, + )?; let batch = Batch::try_new( input, ["a", "a", "b", "b"] diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 32804d4be..e1bc2c82b 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -788,3 +788,53 @@ async fn counter_rates_over_a_shared_query_time_chain_answer_as_installed() { } } } + +// ── Global aggregates over an aggregate's output ───────────────────────────── + +/// A PromQL aggregate with no grouping keys over another aggregate's output +/// is still a PromQL aggregate: over an empty vector it answers no sample, +/// as Prometheus does, not the one row (0) a SQL `SELECT count(*)` has over +/// no rows. Every value of `m` lies above 0 and below 1,000 ([`m_samples`]), +/// so `count(sum by (job) (m >= 1000))` counts an empty vector while +/// `count(sum by (job) (m >= 0))` counts the 2 jobs, and +/// `count(count by (x) (m))` counts one group, as no series has `x`. With +/// no sample at all every query is empty. Each query is planned on its +/// own, exactly. +#[tokio::test] +async fn a_global_promql_count_over_an_empty_aggregate_answers_no_sample() { + // The query's answer values over `samples` (an answer row has no labels). + let values = |plan: &InstallablePlan, samples: &[RawDataSample]| -> Vec { + let query_time = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + run( + &plan.dag, + time_series_tables(&plan.dag, samples), + &[], + &query_roots(plan), + query_time, + ) + .concat() + .iter() + .flat_map(Batch::to_rows) + .map(|row| match row.as_slice() { + [Value::Int64(count)] => *count as f64, + [Value::Float64(value)] => *value, + other => panic!("an answer row {other:?}"), + }) + .collect() + }; + for (query, with_samples) in [ + ("count(sum by (job) (m >= 1000))", vec![]), + ("count(count by (x) (m))", vec![1.0]), + ("count(sum by (job) (m >= 0))", vec![2.0]), + ] { + let plan = plan_every_minute(&[(query, "")]).await; + assert_eq!( + values(&plan, &[]), + Vec::::new(), + "{query} with no sample" + ); + assert_eq!(values(&plan, &m_samples()), with_samples, "{query}"); + } +} diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 309a83ace..23eaf636a 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -147,6 +147,7 @@ fn shown(value: &Value) -> String { match value { Value::Int64(n) => format!("Int64({n})"), Value::Float64(x) => format!("Float64({x})"), + Value::Null => "NULL".to_string(), other => format!("{other:?}"), } } @@ -439,3 +440,39 @@ async fn a_window_equal_to_its_interval_is_built_at_ingestion_time_from_table_ro .collect(); assert_eq!(by_key(run_data_plane(&plan).remove(0)), expected); } + +/// A global SQL aggregate has one row even over no input row, as SQL +/// requires: over the last 5 minutes with `WHERE false`, `COUNT(*)` is 0 +/// and `SUM` is NULL. Over the last 5 minutes of [`rows`] (`latency` = +/// 900, 901, …, 1199) they are 300 and 314,850. +#[tokio::test] +async fn a_global_sql_aggregate_over_no_row_answers_one_row() { + let last_5_minutes = "FROM http WHERE ts >= now() - INTERVAL '5 minutes'"; + let queries = [ + format!("SELECT count(*) {last_5_minutes} AND false"), + format!("SELECT sum(latency) {last_5_minutes} AND false"), + format!("SELECT count(*) {last_5_minutes}"), + format!("SELECT sum(latency) {last_5_minutes}"), + ]; + let queries = queries.each_ref().map(String::as_str); + let plan = plan_for( + &queries, + EXACT, + MINUTE_MS as u64, + &DeploymentCapabilities::default(), + ) + .await; + let answers: Vec>> = run_data_plane(&plan) + .into_iter() + .map(|rows| { + rows.into_iter() + .map(|row| row.iter().map(shown).collect()) + .collect() + }) + .collect(); + assert_eq!( + answers, + ["Int64(0)", "NULL", "Int64(300)", "Float64(314850)",] + .map(|value| vec![vec![value.to_string()]]) + ); +} From ddd82a108a86465878ac5085ccb5df9903f8de0d Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:30:18 +0000 Subject: [PATCH 2/4] refactor(executor): read the IR's rows kind; one empty-global setter and NULL list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The executor's `rows_kind` copied the IR's private `source_kind` and stopped at states built through `SummarySubtract`, `SummaryJoin`, `MaintainPopulation` or `Extension`, so a global SQL rollup of such states got PromQL's rule (no row over no input rows). The IR now exposes `evaluated_rows_kind` (a node's own kind, or for summary states the kind of the rows they were built from) and `input_rows_kind` calls it. `compile_internal` computes `over_relation` once, only for aggregates, summary builds and merges, and passes that one bool form through `compile_node`, `bind_operation` and `summary_agg_build`. `with_emit_empty_global` takes the flag (false leaves the operator as it is), replacing five `if … else Ok(op)` sites, and `Reduction::is_null_without_values` is the one list of measures that are NULL for a group with no value, read by it and by `with_measure_filters`. --- .../executor/src/operators/aggregate/mod.rs | 27 +++-- crates/executor/src/operators/mod.rs | 24 ++-- crates/executor/src/operators/unchecked.rs | 50 +++------ crates/executor/src/physical_planner/mod.rs | 103 +++++++----------- crates/executor/src/physical_planner/tests.rs | 67 ++++++++++++ crates/executor/tests/hydra_cms_execution.rs | 45 +++----- .../tests/keyed_summary_row_weight.rs | 7 +- crates/executor/tests/physical_dag.rs | 18 +-- crates/executor/tests/physical_semantics.rs | 90 +++------------ crates/executor/tests/promql_binary.rs | 7 +- crates/executor/tests/univmon_execution.rs | 6 +- crates/types/src/ir/operator/asap.rs | 13 +++ 12 files changed, 206 insertions(+), 251 deletions(-) diff --git a/crates/executor/src/operators/aggregate/mod.rs b/crates/executor/src/operators/aggregate/mod.rs index 6e3956f4b..855b7e627 100644 --- a/crates/executor/src/operators/aggregate/mod.rs +++ b/crates/executor/src/operators/aggregate/mod.rs @@ -151,16 +151,7 @@ impl Operator { if filter.dtype(&self.inputs[0])?.0 != DataType::Bool { return Err(invalid("measure filter must be boolean")); } - if matches!( - measures[index], - Reduction::Sum(_) - | Reduction::Avg(_) - | Reduction::Min(_) - | Reduction::Max(_) - | Reduction::Percentile { .. } - | Reduction::StdDev { .. } - | Reduction::Variance { .. } - ) { + if measures[index].is_null_without_values() { output.fields[groups.len() + index].nullable = true; } } @@ -293,6 +284,22 @@ pub enum Reduction { /// PromQL `group`: 1 for every group. Group, } +impl Reduction { + /// Whether the measure is NULL for a group with no (matching) value, as + /// SQL `SELECT sum(x) FROM t WHERE false` is NULL while `count(*)` is 0. + pub(super) fn is_null_without_values(&self) -> bool { + matches!( + self, + Reduction::Sum(_) + | Reduction::Avg(_) + | Reduction::Min(_) + | Reduction::Max(_) + | Reduction::Percentile { .. } + | Reduction::StdDev { .. } + | Reduction::Variance { .. } + ) + } +} pub(super) fn execute<'a>( operator: &'a Operator, mut inputs: Vec>, diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index f9417a739..5f87ac3e7 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -364,11 +364,12 @@ impl Operator { self.output.clone() } - /// Give an aggregate with no groups SQL's rule for an empty input: its - /// one global group still has a row, as `SELECT count(*) FROM t WHERE - /// false` answers 0 and `SELECT sum(x) FROM t WHERE false` NULL. Without - /// it (the default) an empty input has no row, as a PromQL `count(v)` - /// over an empty vector has no sample. + /// An aggregate, per-group build or merge into no groups follows SQL's + /// rule for an empty input when `emit_empty_global`: its one global + /// group still has a row, as `SELECT count(*) FROM t WHERE false` + /// answers 0 and `SELECT sum(x) FROM t WHERE false` NULL. Otherwise (the + /// default; `false` leaves the operator as it is) an empty input has no + /// row, as a PromQL `count(v)` over an empty vector has no sample. /// /// An `Aggregate`'s SUM, AVG, MIN, MAX and percentile measures become /// nullable (NULL over no row). A `PerGroupSummaryBuild` outputs an empty @@ -377,8 +378,12 @@ impl Operator { /// `None` for an exact family and for the other two. pub fn with_emit_empty_global( mut self, + emit_empty_global: bool, parameters: Option, ) -> Result { + if !emit_empty_global { + return Ok(self); + } match &mut self.kind { Kind::Aggregate { groups, @@ -388,14 +393,7 @@ impl Operator { } if groups.is_empty() && parameters.is_none() => { let mut output = (*self.output).clone(); for (field, measure) in output.fields.iter_mut().zip(measures.iter()) { - if matches!( - measure, - Reduction::Sum(_) - | Reduction::Avg(_) - | Reduction::Min(_) - | Reduction::Max(_) - | Reduction::Percentile { .. } - ) { + if measure.is_null_without_values() { field.nullable = true; } } diff --git a/crates/executor/src/operators/unchecked.rs b/crates/executor/src/operators/unchecked.rs index 21796d5f5..dae78bb3e 100644 --- a/crates/executor/src/operators/unchecked.rs +++ b/crates/executor/src/operators/unchecked.rs @@ -158,14 +158,9 @@ impl TryFrom for Operator { return Err(invalid("aggregate width mismatch")); } let names = output.fields[groups.len()..].iter().map(|f| f.name.clone()); - let aggregate = - Operator::aggregate(input(0)?, groups, names.zip(measures).collect())? - .with_measure_filters(filters)?; - if emit_empty_global { - aggregate.with_emit_empty_global(None)? - } else { - aggregate - } + Operator::aggregate(input(0)?, groups, names.zip(measures).collect())? + .with_measure_filters(filters)? + .with_emit_empty_global(emit_empty_global, None)? } Kind::SemiJoin { keys, @@ -198,24 +193,18 @@ impl TryFrom for Operator { groups, filter, emit_empty_global, - } => { - let build = with_filter( - Operator::per_group_summary_build( - input(0)?, - family, - parameters, - value, - time, - groups, - )?, - filter, - )?; - if emit_empty_global { - build.with_emit_empty_global(None)? - } else { - build - } - } + } => with_filter( + Operator::per_group_summary_build( + input(0)?, + family, + parameters, + value, + time, + groups, + )?, + filter, + )? + .with_emit_empty_global(emit_empty_global, None)?, Kind::PerGroupKeyedSummaryBuild { family, parameters, @@ -261,16 +250,13 @@ impl TryFrom for Operator { emit_empty_global, parameters, } => { - let merge = Operator::summary_merge(input(0)?, state, groups)?; - if emit_empty_global { - merge.with_emit_empty_global(parameters)? - } else if parameters.is_some() { + if !emit_empty_global && parameters.is_some() { return Err(invalid( "only a merge that outputs an empty global state takes parameters θ", )); - } else { - merge } + Operator::summary_merge(input(0)?, state, groups)? + .with_emit_empty_global(emit_empty_global, parameters)? } Kind::Evaluation { state, query } => Operator::evaluation(input(0)?, state, query)?, } diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index ebecd8c0f..724a44f8b 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -10,6 +10,7 @@ use crate::{ values::{Batch, SchemaRef}, Error, }; +use planner_types::ir::operator::asap::evaluated_rows_kind; use planner_types::ir::operator::{ agg_has_exact_per_entity_kernel, AggIntent, GroupKeys, Reduction as PlannerReduction, }; @@ -258,8 +259,20 @@ fn compile_internal( .iter() .map(|id| Arc::new(nodes[id].output_schema.clone())) .collect::>(); - let input_rows = input_rows_kind(&restored[&id]); - let over_relation = input_rows == OperatorResultKind::Relation; + // Whether an aggregate, summary build or merge reads a SQL + // relation's rows (or states built from them), which gives it + // SQL's rules for an empty input and a quantile; no other + // operator reads it. + let over_relation = matches!( + node.payload, + Payload::NonASAP(NonASAPOp::Aggregate { .. }) + | Payload::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryMerge { .. } + ) + ) && input_rows_kind(&restored[&id]) + == OperatorResultKind::Relation; if matches!(node.payload, Payload::ASAP(ASAPOp::SummaryMerge { .. })) && inputs.len() > 1 { @@ -885,7 +898,7 @@ fn compile_internal( physical_dag.add(id, inputs, operator.with_output_schema(output)?)?; continue; } - let mut operator = compile_node(node, &schemas, input_rows) + let mut operator = compile_node(node, &schemas, over_relation) .map_err(|error| invalid(format!("node {id}: {error}")))?; if operator.is_counter_evaluation() { if let Some((offset, lookback)) = counter_window(id, &nodes, &dependencies)? { @@ -905,37 +918,18 @@ fn compile_internal( } /// The result kind of the rows `node` reads: its (first) input's, or, when -/// that input is summary states, the kind of the rows they were built from. -/// It decides an aggregate's rule for an empty input: a global aggregate -/// over a SQL table's `Relation` has one row over no input rows -/// (`SELECT count(*) FROM t WHERE false` is 0), over a PromQL vector it has -/// none (`count(sum by (job) (m >= 1000))` is empty when no value reaches -/// 1000), whether or not its input still has a series identity column. +/// that input is summary states, the kind of the rows they were built from +/// ([`evaluated_rows_kind`]). It decides an aggregate's rule for an empty +/// input: a global aggregate over a SQL table's `Relation` has one row over +/// no input rows (`SELECT count(*) FROM t WHERE false` is 0), over a PromQL +/// vector it has none (`count(sum by (job) (m >= 1000))` is empty when no +/// value reaches 1000), whether or not its input still has a series +/// identity column. fn input_rows_kind(node: &OperatorNode) -> OperatorResultKind { node.operator .children() .first() - .map_or(node.result_kind, |input| rows_kind(input)) -} - -/// The result kind of `node`'s rows, or, for summary states, of the rows -/// they were built from (`State` when that is not known). -fn rows_kind(node: &std::rc::Rc) -> OperatorResultKind { - if node.result_kind != OperatorResultKind::State { - return node.result_kind; - } - match node.asap() { - Some( - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => rows_kind(child), - // The states of one memo group are built from rows of one kind. - Some(ASAPOp::SummaryMerge { children, .. }) => { - children.first().map_or(node.result_kind, rows_kind) - } - // A filter keeping whole states reads them from their build. - _ => planner_types::ir::operator::asap::selected_from(node) - .map_or(node.result_kind, |states| rows_kind(&states)), - } + .map_or(node.result_kind, |input| evaluated_rows_kind(*input)) } // Temporal summary evaluations produce PromQL vectors, whose range functions drop @@ -980,25 +974,22 @@ pub(crate) fn require_sound_merge(input: &SchemaRef) -> Result<(), Error> { /// that lowers to one operator. A node that lowers to several (an exact /// top-k, `count_values`, or a `without` grouping over a series identity) /// is lowered only by [`compile`]; here it is refused. -/// `input_rows_kind` is the result kind of the rows the node reads (for -/// summary states, of the rows they were built from): an aggregate, build -/// or merge into no groups over a `Relation` has SQL's one row over no -/// input rows, and its quantile is SQL's percentile; over a PromQL vector -/// it has no row and PromQL's quantile. +/// `over_relation`: whether the node reads a SQL relation's rows (for +/// summary states, whether they were built from one). Then an aggregate, +/// build or merge into no groups has SQL's one row over no input rows, and +/// a quantile is SQL's percentile. Otherwise (a PromQL vector, a range +/// vector, or states built from either) it has no row and PromQL's +/// quantile. pub fn compile_node( node: &PhysicalASAPDAGNode, inputs: &[SchemaRef], - input_rows_kind: OperatorResultKind, + over_relation: bool, ) -> Result { for schema in inputs { crate::values::validate_schema(schema)?; } - bind_operation( - node, - inputs, - input_rows_kind == OperatorResultKind::Relation, - )? - .with_output_schema(Arc::new(node.output_schema.clone())) + bind_operation(node, inputs, over_relation)? + .with_output_schema(Arc::new(node.output_schema.clone())) } /// `over_relation`: whether the node reads a SQL relation's rows (or @@ -1314,18 +1305,14 @@ fn bind_operation( .collect::>()?; let groups = groups(input, keys)?; let global = groups.is_empty(); - let aggregate = Operator::aggregate(input.clone(), groups, measures)? + Operator::aggregate(input.clone(), groups, measures)? .with_measure_filters( filters .iter() .map(|f| f.as_ref().map(|f| expression(&f.0, input)).transpose()) .collect::>()?, - )?; - if global && over_relation { - aggregate.with_emit_empty_global(None) - } else { - Ok(aggregate) - } + )? + .with_emit_empty_global(global && over_relation, None) } _ => Err(invalid("value operation has no native implementation")), }, @@ -1380,12 +1367,8 @@ fn bind_operation( // aggregate: one row, an empty state of the θ the merge // carries (the states'), over no input rows. let global = groups.is_empty() && group_by.is_some(); - let merge = Operator::summary_merge(input.clone(), state, groups)?; - if global && over_relation { - merge.with_emit_empty_global(node.parameters.clone()) - } else { - Ok(merge) - } + Operator::summary_merge(input.clone(), state, groups)? + .with_emit_empty_global(global && over_relation, node.parameters.clone()) } Payload::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { if let SketchStatistic::TopK { k } = query { @@ -1443,19 +1426,15 @@ fn summary_agg_build( // A state per group of `groups` over `value`. let per_group = |value: usize, groups: Vec| -> Result { let global = groups.is_empty(); - let build = Operator::per_group_summary_build( + Operator::per_group_summary_build( input.clone(), family.clone(), parameters.cloned(), value, input.time_index, groups, - )?; - if global && over_relation { - build.with_emit_empty_global(None) - } else { - Ok(build) - } + )? + .with_emit_empty_global(global && over_relation, None) }; if matches!( family, diff --git a/crates/executor/src/physical_planner/tests.rs b/crates/executor/src/physical_planner/tests.rs index fcf065b20..b9514250f 100644 --- a/crates/executor/src/physical_planner/tests.rs +++ b/crates/executor/src/physical_planner/tests.rs @@ -75,3 +75,70 @@ fn a_stored_time_shift_over_a_chain_reads_the_chain_unbounded() { assert_eq!(window_read_above(0, &graph), None); } } + +/// A global merge (`group_by: Some([])`) of summary states built from a SQL +/// table's rows reads a `Relation`, so it keeps SQL's one row over no input +/// rows, also when another summary operator sits between the build and the +/// merge: `SummaryMerge([]) ← SummarySubtract ← SummaryAgg ← Scan(t)` and +/// the same with `SummaryJoin`. Over a PromQL series scan the same tree +/// reads an instant vector (no row over an empty vector). +#[test] +fn a_global_merge_reads_the_rows_its_states_were_built_from() { + use planner_types::ir::operator::arguments::Source; + use planner_types::ir::operator::Reduction as IrReduction; + use planner_types::ir::schema::Schema; + use planner_types::ir::Operator as IrOperator; + use planner_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; + use std::rc::Rc; + let node = |operator| Rc::new(OperatorNode::with_schema(operator, Schema::default())); + for (source, expected) in [ + ( + Source::Table { + table_ref: "t".into(), + }, + OperatorResultKind::Relation, + ), + ( + Source::TimeSeries { metric: "m".into() }, + OperatorResultKind::InstantVector, + ), + ] { + let scan = node(IrOperator::NonASAP(NonASAPOp::Scan { + source, + predicates: vec![], + schema: Schema::default(), + })); + let build = node(IrOperator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: IrReduction::PerEntity, + filter: None, + })); + let subtract = node(IrOperator::ASAP(ASAPOp::SummarySubtract { + left: build.clone(), + right: build.clone(), + })); + let join = node(IrOperator::ASAP(ASAPOp::SummaryJoin { + outer: build.clone(), + inner: build.clone(), + key: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + })); + for states in [subtract, join] { + let merge = OperatorNode::with_schema( + IrOperator::ASAP(ASAPOp::SummaryMerge { + children: vec![states.clone()], + group_by: Some(vec![]), + }), + Schema::default(), + ); + assert_eq!( + input_rows_kind(&merge), + expected, + "{}", + states.operator.kind_name() + ); + } + } +} diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 32b690814..5b72f4a04 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -171,7 +171,7 @@ fn try_run( let build = compile_node( &summary_agg(update, filter), std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, + true, ) .unwrap(); let state = build.schema(); @@ -204,7 +204,7 @@ fn try_run( (*state).clone(), ), std::slice::from_ref(&state), - planner_types::ir::OperatorResultKind::Relation, + true, ) .unwrap(); operators.insert(201, (vec![200], merge)); @@ -213,7 +213,7 @@ fn try_run( let read = compile_node( &estimate(3, point("checkout"), DataType::Float64), std::slice::from_ref(&state), - planner_types::ir::OperatorResultKind::Relation, + true, ) .unwrap(); operators.insert(300, (vec![merged], read)); @@ -337,7 +337,7 @@ fn positive_constant_weight_adds_the_constant_per_row() { let error = compile_node( &summary_agg(constant(weight), None), &[input_schema()], - planner_types::ir::OperatorResultKind::Relation, + true, ) .err() .unwrap_or_else(|| panic!("constant weight {weight} accepted")); @@ -360,12 +360,7 @@ fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { (constant, serde_json::json!({"Constant": 2})), (column, serde_json::json!({"Column": 2})), ] { - let build = compile_node( - &summary_agg(update, None), - &[input_schema()], - planner_types::ir::OperatorResultKind::Relation, - ) - .unwrap(); + let build = compile_node(&summary_agg(update, None), &[input_schema()], true).unwrap(); let wire = serde_json::to_value(&build).unwrap(); let fields = &wire["kind"]["AllGroupsSummaryBuild"]; assert_eq!(fields["row_weight"], expected, "{wire}"); @@ -415,13 +410,9 @@ fn constant_weight_above_i32_is_refused_at_validation() { error.contains("HydraCms") && error.contains("i32::MAX"), "{error}" ); - let error = compile_node( - &summary_agg(update, None), - &[input_schema()], - planner_types::ir::OperatorResultKind::Relation, - ) - .err() - .expect("constant above i32::MAX compiled"); + let error = compile_node(&summary_agg(update, None), &[input_schema()], true) + .err() + .expect("constant above i32::MAX compiled"); assert!( error.to_string().contains("HydraCms") && error.to_string().contains("i32::MAX"), "{error}" @@ -451,24 +442,16 @@ fn invalid_weight_values_fail_the_build() { // rather than computed from sketchlib internals. #[test] fn group_total_is_rejected_until_sketchlib_offers_it() { - let state = compile_node( - &summary_agg(unit_count(), None), - &[input_schema()], - planner_types::ir::OperatorResultKind::Relation, - ) - .unwrap() - .schema(); + let state = compile_node(&summary_agg(unit_count(), None), &[input_schema()], true) + .unwrap() + .schema(); let total = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, }; - let error = compile_node( - &estimate(3, total, DataType::Int64), - &[state], - planner_types::ir::OperatorResultKind::Relation, - ) - .err() - .expect("group total"); + let error = compile_node(&estimate(3, total, DataType::Int64), &[state], true) + .err() + .expect("group total"); assert!( error .to_string() diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index cd654507b..92a7300b5 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -175,12 +175,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { selected_physical_candidate: None, view_coverage: None, }; - let build = compile_node( - &node, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - ) - .unwrap(); + let build = compile_node(&node, std::slice::from_ref(&input), true).unwrap(); let wire = serde_json::to_value(&build).unwrap(); assert_eq!( wire["kind"]["PerGroupKeyedSummaryBuild"]["row_weight"], diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 15d55d361..95013aa3d 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -307,7 +307,7 @@ fn exact_integer_and_empty_extrema() { vec![0], Operator::aggregate(schema, vec![], vec![("min".into(), Reduction::Min(0))]) .unwrap() - .with_emit_empty_global(None) + .with_emit_empty_global(true, None) .unwrap(), Lifetime::None, ) @@ -663,7 +663,7 @@ fn empty_exact_count_is_an_integer_state_evaluation() { vec![], ) .unwrap() - .with_emit_empty_global(None) + .with_emit_empty_global(true, None) .unwrap(); let read = Operator::evaluation( build.schema(), @@ -1307,12 +1307,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { let join = if certified { Operator::certified_semi_join(schema.clone(), schema.clone(), vec![(0, 0)]).unwrap() } else { - compile_node( - &node, - &[schema.clone(), schema.clone()], - planner_types::ir::OperatorResultKind::Relation, - ) - .unwrap() + compile_node(&node, &[schema.clone(), schema.clone()], true).unwrap() }; // Each side is a leaf holding its keys. let plan = |left: Vec<&str>, right: Vec<&str>| { @@ -1416,12 +1411,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { selected_physical_candidate: None, view_coverage: None, }; - let binary = compile_node( - &node, - &[input.clone(), input.clone()], - planner_types::ir::OperatorResultKind::InstantVector, - ) - .unwrap(); + let binary = compile_node(&node, &[input.clone(), input.clone()], false).unwrap(); for (right, expected) in [ (vec![("b", 2, 3.), ("a", 1, 2.)], Some(vec![8., 17.])), (vec![("b", 2, 3.)], None), diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 68fae56cf..0c808dc05 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -750,7 +750,7 @@ fn aggregate_empty_and_all_null_follow_asap_contract() { ], ) .unwrap() - .with_emit_empty_global(None) + .with_emit_empty_global(true, None) .unwrap(); let result = unary(input.clone(), batches, op); assert_eq!(result.len(), 1); @@ -826,12 +826,8 @@ fn global_extrema_bind_with_planner_derived_schema() { selected_physical_candidate: None, view_coverage: None, }; - let operator = compile_node( - &node, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - ) - .expect("global extremum should bind to its Planner schema"); + let operator = compile_node(&node, std::slice::from_ref(&input), true) + .expect("global extremum should bind to its Planner schema"); assert!(operator.schema().fields[0].nullable); let empty = unary(input.clone(), vec![], operator.clone()); assert!(matches!(empty[0][0], Value::Null)); @@ -1328,7 +1324,7 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { vec![], ) .unwrap() - .with_emit_empty_global(None) + .with_emit_empty_global(true, None) .unwrap(); let state = build.schema(); let mut dag = PhysicalDAG::default(); @@ -1475,14 +1471,7 @@ fn measure_filters_keep_groups_without_matching_rows() { ("all", plain(DataType::Int64), false), ], ); - let operator = round_trip( - compile_node( - &node, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - ) - .unwrap(), - ); + let operator = round_trip(compile_node(&node, std::slice::from_ref(&input), true).unwrap()); assert_eq!( printed(&unary(input, vec![rows], operator)), ["a 1 1.0 2", "b 0 NULL 1", "c 1 4.0 1"] @@ -1510,12 +1499,7 @@ fn filtered_sum_declared_non_null_is_rejected() { ("s", FieldDataType::Plain(DataType::Float64), false), ], ); - assert!(compile_node( - &node, - &[input], - planner_types::ir::OperatorResultKind::Relation - ) - .is_err()); + assert!(compile_node(&node, &[input], true).is_err()); } /// A filtered `SummaryAgg` updates its state only from kept rows but keeps @@ -1566,14 +1550,7 @@ fn filtered_summary_build_keeps_empty_groups() { ], ); asap_executor::physical_planner::test_support::set_build_parameters([&mut build], None); - let build = round_trip( - compile_node( - &build, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - ) - .unwrap(), - ); + let build = round_trip(compile_node(&build, std::slice::from_ref(&input), true).unwrap()); let read = planner_node( PhysicalASAPOperatorPayload::ASAP(read), &[ @@ -1581,12 +1558,7 @@ fn filtered_summary_build_keeps_empty_groups() { ("value", FieldDataType::Plain(result), nullable), ], ); - let read = compile_node( - &read, - &[build.schema()], - planner_types::ir::OperatorResultKind::Relation, - ) - .unwrap(); + let read = compile_node(&read, &[build.schema()], true).unwrap(); let mut dag = PhysicalDAG::default(); dag.add( 0, @@ -1632,11 +1604,7 @@ fn filtered_summary_build_is_rejected_at_ingestion_time() { ], ); node.output_state = ExecutionDataState::INGESTION_SUMMARY; - compile_node( - &node, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - ) + compile_node(&node, std::slice::from_ref(&input), true) }; assert!(build(None).is_ok()); let error = build(keep()).err().expect("filtered ingestion-time build"); @@ -1714,7 +1682,6 @@ fn exact_aggregate_node( fn a_global_aggregate_follows_its_input_rows_kind() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::AggIntent; - use planner_types::ir::OperatorResultKind; let input = schema(&[ ("job", DataType::Utf8, false), @@ -1739,12 +1706,8 @@ fn a_global_aggregate_follows_its_input_rows_kind() { ] { let mut node = exact_aggregate_node(measure.clone(), "value", DataType::Float64); // A PromQL vector: non-null, no sample over an empty vector. - let promql = compile_node( - &node, - std::slice::from_ref(&input), - OperatorResultKind::InstantVector, - ) - .unwrap_or_else(|error| panic!("{measure:?}: {error}")); + let promql = compile_node(&node, std::slice::from_ref(&input), false) + .unwrap_or_else(|error| panic!("{measure:?}: {error}")); assert!( unary(input.clone(), vec![], promql.clone()).is_empty(), "{measure:?}" @@ -1755,21 +1718,12 @@ fn a_global_aggregate_follows_its_input_rows_kind() { "{measure:?}: {answer:?}" ); // A SQL relation: NULL over no row, so a non-null output is refused. - assert!(compile_node( - &node, - std::slice::from_ref(&input), - OperatorResultKind::Relation - ) - .is_err()); + assert!(compile_node(&node, std::slice::from_ref(&input), true).is_err()); node.output_schema.fields[0].nullable = true; // The rule survives serialization. let sql = round_trip( - compile_node( - &node, - std::slice::from_ref(&input), - OperatorResultKind::Relation, - ) - .unwrap_or_else(|error| panic!("{measure:?}: {error}")), + compile_node(&node, std::slice::from_ref(&input), true) + .unwrap_or_else(|error| panic!("{measure:?}: {error}")), ); let empty = unary(input.clone(), vec![], sql.clone()); assert!( @@ -1823,12 +1777,8 @@ fn exact_frequency_intents_execute_typed_keys_and_empty_input() { ), ] { let node = exact_aggregate_node(measure, name, DataType::Float64); - let operator = compile_node( - &node, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - ) - .expect("exact frequency intent binds"); + let operator = compile_node(&node, std::slice::from_ref(&input), true) + .expect("exact frequency intent binds"); let rows = values .iter() .flat_map(|v| [vec![v.clone()], vec![v.clone()]]) @@ -1932,12 +1882,8 @@ fn exact_cardinality_binds_and_executes_typed_tuples() { for (cols, expected) in [(vec![0], 2), (vec![0, 1], 3)] { let node = exact_aggregate_node(AggIntent::Cardinality { cols }, "distinct", DataType::Int64); - let operator = compile_node( - &node, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - ) - .expect("exact distinct intent binds"); + let operator = compile_node(&node, std::slice::from_ref(&input), true) + .expect("exact distinct intent binds"); let rows = vec![ vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 03efa35a9..8954eeee9 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -86,12 +86,7 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { selected_physical_candidate: None, view_coverage: None, }; - compile_node( - &node, - &[schema.clone(), schema], - planner_types::ir::OperatorResultKind::InstantVector, - ) - .unwrap() + compile_node(&node, &[schema.clone(), schema], false).unwrap() } /// `binary` (node 2) over the sources `left` (node 0) and `right` (node 1), /// bound to a store with no data: the sources hold their rows. diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 7c1610126..63e9c8237 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -199,11 +199,7 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() selected_physical_candidate: None, view_coverage: None, }; - let build = compile_node( - &node, - std::slice::from_ref(&input), - planner_types::ir::OperatorResultKind::Relation, - )?; + let build = compile_node(&node, std::slice::from_ref(&input), true)?; let batch = Batch::try_new( input, ["a", "a", "b", "b"] diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 44279f9de..4376a0553 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -961,6 +961,19 @@ pub fn summary_type_of_operator(operator: &Operator) -> Option<&FieldDataT } } +/// The category of the rows `node` stands for once evaluated: its own result +/// kind, or, for summary states, the kind of the rows they were built from +/// ([`source_kind`]). Sum states built over a SQL table `t` and then +/// subtracted or merged stand for a `Relation`, so a global rollup of them +/// has SQL's one row over no input rows; built over a PromQL series `m` they +/// stand for an `InstantVector`, so it has no sample over an empty vector. +pub fn evaluated_rows_kind(node: &I) -> OperatorResultKind { + match node.result_kind() { + OperatorResultKind::State => source_kind(node), + kind => kind, + } +} + /// The category of the values a evaluation of `state` produces: the category /// of the relational input the state was built from. fn source_kind(node: &I) -> OperatorResultKind { From d147539087ac295d48568442797e0ba4de119be6 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 03:30:41 +0000 Subject: [PATCH 3/4] fix(executor): global stddev and stdvar follow the input's result kind Grouped stddev/stdvar still read the series identity column to choose between SQL and PromQL behaviour over no value. They now follow the same rule as the other global aggregates: the measure is nullable for a sample form or a nullable column, a global SQL aggregate gets its one NULL row from `with_emit_empty_global`, and a PromQL `stddev(...)`/`stdvar(...)` over an empty vector answers no sample. Tests: the stddev and stdvar cases of a_global_aggregate_follows_its_input_rows_kind (failed before: the derived output was nullable over a PromQL vector without an identity column), PromQL stddev/stdvar over an empty aggregate, SQL stddev_pop and var_samp over no row. The SQL variance and correlation tests build their global aggregate with `with_emit_empty_global`; the PromQL install test uses the shared `run` over a store. --- .../src/operators/aggregate/columnar.rs | 1 - .../executor/src/operators/aggregate/mod.rs | 15 ++++----- crates/executor/src/operators/mod.rs | 4 +-- crates/executor/tests/physical_semantics.rs | 32 ++++++++++++++++--- crates/planner/tests/runtime_install.rs | 8 +++-- crates/planner/tests/sql_table_scans.rs | 15 +++++++-- 6 files changed, 54 insertions(+), 21 deletions(-) diff --git a/crates/executor/src/operators/aggregate/columnar.rs b/crates/executor/src/operators/aggregate/columnar.rs index 372319bcd..a0897d269 100644 --- a/crates/executor/src/operators/aggregate/columnar.rs +++ b/crates/executor/src/operators/aggregate/columnar.rs @@ -616,7 +616,6 @@ fn finish_measure( }; match value { Some(value) => Value::Float64(value), - None if promql => Value::Float64(f64::NAN), None => Value::Null, } }) diff --git a/crates/executor/src/operators/aggregate/mod.rs b/crates/executor/src/operators/aggregate/mod.rs index 855b7e627..8026d5cbd 100644 --- a/crates/executor/src/operators/aggregate/mod.rs +++ b/crates/executor/src/operators/aggregate/mod.rs @@ -81,15 +81,14 @@ impl Operator { if !matches!(t, DataType::Int64 | DataType::Float64) { return Err(invalid("numeric aggregate input required")); } - // SQL: NULL for fewer than two values (sample) or none - // (population); a PromQL group always has a value. The - // IR's aggregate schema defines this nullability; this + // NULL for a group of NULLs and, as SQL's sample + // `STDDEV`/`VAR_SAMP`, for a group of one value (and for + // a global one over no row: `with_emit_empty_global`). + // PromQL's `stddev`/`stdvar` are population ones over + // groups that have a value, so never NULL. The IR's + // aggregate schema defines this nullability; this // derives it again to check the declared output. - ( - DataType::Float64, - !input.has_promql_series_identity() - && (!population || groups.is_empty() || nullable), - ) + (DataType::Float64, !population || nullable) } Reduction::PearsonCorr { left, right } => { for column in [left, right] { diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index 5f87ac3e7..8141ab4b0 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -371,8 +371,8 @@ impl Operator { /// default; `false` leaves the operator as it is) an empty input has no /// row, as a PromQL `count(v)` over an empty vector has no sample. /// - /// An `Aggregate`'s SUM, AVG, MIN, MAX and percentile measures become - /// nullable (NULL over no row). A `PerGroupSummaryBuild` outputs an empty + /// An `Aggregate`'s SUM, AVG, MIN, MAX, percentile, standard deviation + /// and variance measures become nullable (NULL over no row). A `PerGroupSummaryBuild` outputs an empty /// state of its own θ, and a `SummaryMerge` one of `parameters`: the θ /// of the states it merges (the exported build node's `parameters`), /// `None` for an exact family and for the other two. diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 0c808dc05..19d2d0715 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -1674,10 +1674,11 @@ fn exact_aggregate_node( // A global aggregate follows the result kind of the rows it reads, not // whether they carry a series identity column. Over `[job, v]`, the output // of `max by (job) (m)` (no identity column), PromQL's `sum(…)`, `avg`, -// `min`, `max` and `quantile(0.5, …)` are non-null and answer no sample over -// an empty vector; over the same columns as a SQL relation they are -// nullable and answer one NULL row over no row. Over jobs a (1) and b (3) -// both answer 4, 2, 1, 3 and 2. +// `min`, `max`, `quantile(0.5, …)`, `stddev` and `stdvar` are non-null and +// answer no sample over an empty vector; over the same columns as a SQL +// relation they are nullable and answer one NULL row over no row. Over jobs +// a (1) and b (3) both answer 4, 2, 1, 3, 2, 1 and 1 (mean 2, population +// variance ((1 - 2)² + (3 - 2)²) / 2 = 1). #[test] fn a_global_aggregate_follows_its_input_rows_kind() { use asap_executor::physical_planner::compile_node; @@ -1703,6 +1704,20 @@ fn a_global_aggregate_follows_its_input_rows_kind() { }, 2.0, ), + ( + AggIntent::StdDev { + col: Some(1), + population: true, + }, + 1.0, + ), + ( + AggIntent::Variance { + col: Some(1), + population: true, + }, + 1.0, + ), ] { let mut node = exact_aggregate_node(measure.clone(), "value", DataType::Float64); // A PromQL vector: non-null, no sample over an empty vector. @@ -1994,7 +2009,11 @@ fn sql_variance_and_stddev_follow_their_definitions() { ] }; let run = |values: &[Option]| { - let op = Operator::aggregate(input.clone(), vec![], measures()).unwrap(); + // A global SQL aggregate: one row also over no input row. + let op = Operator::aggregate(input.clone(), vec![], measures()) + .unwrap() + .with_emit_empty_global(true, None) + .unwrap(); let rows = unary(input.clone(), vec![floats(values)], op); (0..4).map(|c| float_at(&rows, 0, c)).collect::>() }; @@ -2103,6 +2122,9 @@ fn correlation_follows_datafusion() { vec![], vec![("corr".into(), Reduction::PearsonCorr { left: 0, right: 1 })], ) + .unwrap() + // A global SQL aggregate: one row also over no input row. + .with_emit_empty_global(true, None) .unwrap(); let rows = pairs .iter() diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index e1bc2c82b..cf94edc90 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -797,8 +797,9 @@ async fn counter_rates_over_a_shared_query_time_chain_answer_as_installed() { /// no rows. Every value of `m` lies above 0 and below 1,000 ([`m_samples`]), /// so `count(sum by (job) (m >= 1000))` counts an empty vector while /// `count(sum by (job) (m >= 0))` counts the 2 jobs, and -/// `count(count by (x) (m))` counts one group, as no series has `x`. With -/// no sample at all every query is empty. Each query is planned on its +/// `count(count by (x) (m))` counts one group, as no series has `x`. +/// `stddev` and `stdvar` follow the same rule: none over the empty vector, +/// and 0 over that one group. With no sample at all every query is empty. Each query is planned on its /// own, exactly. #[tokio::test] async fn a_global_promql_count_over_an_empty_aggregate_answers_no_sample() { @@ -828,6 +829,9 @@ async fn a_global_promql_count_over_an_empty_aggregate_answers_no_sample() { ("count(sum by (job) (m >= 1000))", vec![]), ("count(count by (x) (m))", vec![1.0]), ("count(sum by (job) (m >= 0))", vec![2.0]), + ("stddev(sum by (job) (m >= 1000))", vec![]), + ("stdvar(sum by (job) (m >= 1000))", vec![]), + ("stdvar(count by (x) (m))", vec![0.0]), ] { let plan = plan_every_minute(&[(query, "")]).await; assert_eq!( diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 23eaf636a..1d4d5e7e9 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -443,7 +443,7 @@ async fn a_window_equal_to_its_interval_is_built_at_ingestion_time_from_table_ro /// A global SQL aggregate has one row even over no input row, as SQL /// requires: over the last 5 minutes with `WHERE false`, `COUNT(*)` is 0 -/// and `SUM` is NULL. Over the last 5 minutes of [`rows`] (`latency` = +/// and `SUM`, `STDDEV_POP` and `VAR_SAMP` are NULL. Over the last 5 minutes of [`rows`] (`latency` = /// 900, 901, …, 1199) they are 300 and 314,850. #[tokio::test] async fn a_global_sql_aggregate_over_no_row_answers_one_row() { @@ -451,6 +451,8 @@ async fn a_global_sql_aggregate_over_no_row_answers_one_row() { let queries = [ format!("SELECT count(*) {last_5_minutes} AND false"), format!("SELECT sum(latency) {last_5_minutes} AND false"), + format!("SELECT stddev_pop(latency) {last_5_minutes} AND false"), + format!("SELECT var_samp(latency) {last_5_minutes} AND false"), format!("SELECT count(*) {last_5_minutes}"), format!("SELECT sum(latency) {last_5_minutes}"), ]; @@ -472,7 +474,14 @@ async fn a_global_sql_aggregate_over_no_row_answers_one_row() { .collect(); assert_eq!( answers, - ["Int64(0)", "NULL", "Int64(300)", "Float64(314850)",] - .map(|value| vec![vec![value.to_string()]]) + [ + "Int64(0)", + "NULL", + "NULL", + "NULL", + "Int64(300)", + "Float64(314850)", + ] + .map(|value| vec![vec![value.to_string()]]) ); } From d78f984a5224127cbc652237737e0ba90fee7297 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 16 Nov 2026 04:02:01 +0000 Subject: [PATCH 4/4] test(planner): run the empty-aggregate install test over the plain store Main deleted the read-counting CountingStore; the test reads the plan's raw-data tables directly. --- crates/planner/tests/runtime_install.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index cf94edc90..4fba7b99c 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use crate::time_series; use asap_executor::runtime::Scope; -use asap_executor::values::Batch; +use asap_executor::values::{Batch, Value}; use asap_planner::{ plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, OperatorKind, PlanningModels, PlanningWorkload, SelectedPlanReport, UserInput, @@ -799,8 +799,8 @@ async fn counter_rates_over_a_shared_query_time_chain_answer_as_installed() { /// `count(sum by (job) (m >= 0))` counts the 2 jobs, and /// `count(count by (x) (m))` counts one group, as no series has `x`. /// `stddev` and `stdvar` follow the same rule: none over the empty vector, -/// and 0 over that one group. With no sample at all every query is empty. Each query is planned on its -/// own, exactly. +/// and 0 over that one group. With no sample at all every query is empty. +/// Each query is planned on its own, exactly. #[tokio::test] async fn a_global_promql_count_over_an_empty_aggregate_answers_no_sample() { // The query's answer values over `samples` (an answer row has no labels).