diff --git a/crates/executor/src/operators/aggregate/columnar.rs b/crates/executor/src/operators/aggregate/columnar.rs index 8bf04a1e9..a0897d269 100644 --- a/crates/executor/src/operators/aggregate/columnar.rs +++ b/crates/executor/src/operators/aggregate/columnar.rs @@ -5,10 +5,11 @@ use super::{kahan_inc, percentile_cont, quantile, Reduction}; use crate::operators::common::{row_chunks, Workspace}; use crate::operators::group_table::GroupTable; +use crate::operators::{Kind, Operator}; use crate::runtime::{Cooperative, Input, RunContext}; use crate::values::arrow_columns::{self, ColumnDepth}; use crate::values::{Batch, SchemaRef, Value}; -use crate::{expressions::Expression, Error}; +use crate::Error; use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use arrow::array::{Array, ArrayRef, BooleanArray, Float64Array, Int64Array}; use futures::StreamExt; @@ -491,7 +492,6 @@ fn finish_measure( states: MeasureStates, measure: &Reduction, dtype: &DataType, - promql: bool, order: &[u32], ) -> Result { let values: Vec = match states { @@ -503,7 +503,7 @@ fn finish_measure( .iter() .map(|&g| { let (sum, c, seen) = sums[g as usize]; - if !seen && !promql { + if !seen { Value::Null } else if sum.is_infinite() { Value::Float64(sum) @@ -516,7 +516,7 @@ fn finish_measure( .iter() .map(|&g| { let average = averages[g as usize]; - if average.count == 0 && !promql { + if average.count == 0 { Value::Null } else { Value::Float64(average.value()) @@ -527,7 +527,7 @@ fn finish_measure( .iter() .map(|&g| { let (sum, count) = sums[g as usize]; - if count == 0 && !promql { + if count == 0 { Ok(Value::Null) } else if matches!(measure, Reduction::Avg(_)) { Ok(Value::Float64(sum as f64 / count as f64)) @@ -616,7 +616,6 @@ fn finish_measure( }; match value { Some(value) => Value::Float64(value), - None if promql => Value::Float64(f64::NAN), None => Value::Null, } }) @@ -639,18 +638,25 @@ fn finish_measure( ) } -/// The exact `Aggregate`: the groups of `groups` in key order, each with -/// its `measures` (each over the rows its filter keeps). +/// The exact `Aggregate` `operator`: the groups of its `groups` in key +/// order, each with its `measures` (each over the rows its filter keeps). +/// With `emit_empty_global` and no groups, an empty input still has the +/// global group (SQL). pub(super) async fn aggregate( mut input: Input<'_, Batch>, - groups: &[usize], - measures: &[Reduction], - filters: &[Option], - input_schema: &SchemaRef, - output: &SchemaRef, + operator: &Operator, context: &RunContext, ) -> Result { - let promql = input_schema.has_promql_series_identity(); + let Kind::Aggregate { + groups, + measures, + filters, + emit_empty_global, + } = &operator.kind + else { + unreachable!("an aggregate's run") + }; + let (input_schema, output) = (&operator.inputs[0], &operator.output); // `now()` in a filter is this run's evaluation time. let filters: Vec> = filters .iter() @@ -715,8 +721,9 @@ pub(super) async fn aggregate( workspace.set_bytes(workspace_bytes + table.memory_bytes())?; } } - // PromQL sums over an empty vector emit no sample. - if table.len() == 0 && groups.is_empty() && !promql { + // A global SQL aggregate has one row over no input rows; a PromQL + // aggregate over an empty vector emits no sample. + if table.len() == 0 && groups.is_empty() && *emit_empty_global { table.add_empty_group()?; for state in &mut states { state.ensure_groups(1); @@ -732,7 +739,7 @@ pub(super) async fn aggregate( let FieldDataType::Plain(dtype) = &field.dtype else { return Err(invalid("aggregate measures are plain")); }; - arrays.push(finish_measure(state, measure, dtype, promql, &order)?); + arrays.push(finish_measure(state, measure, dtype, &order)?); } Batch::from_parts( output.clone(), diff --git a/crates/executor/src/operators/aggregate/mod.rs b/crates/executor/src/operators/aggregate/mod.rs index e4b578d36..8026d5cbd 100644 --- a/crates/executor/src/operators/aggregate/mod.rs +++ b/crates/executor/src/operators/aggregate/mod.rs @@ -35,8 +35,10 @@ impl Operator { } (DataType::Float64, false) } + // NULL for a group of NULLs, as SQL `SUM` (and for a + // global one over no row: `with_emit_empty_global`). Reduction::Sum(i) | Reduction::Avg(i) => { - let (t, _) = plain(&input, *i)?; + let (t, nullable) = plain(&input, *i)?; if !matches!(t, DataType::Int64 | DataType::Float64) { return Err(invalid("numeric aggregate input required")); } @@ -46,8 +48,7 @@ impl Operator { } else { t.clone() }, - !input.has_promql_series_identity() - && (groups.is_empty() || plain(&input, *i)?.1), + nullable, ) } Reduction::Quantile { column, .. } => { @@ -63,19 +64,16 @@ impl Operator { "a percentile requires Float64 input and a fraction in [0, 1]", )); } - // NULL for a group with no value, as SQL `SUM`: a global - // one over no row, or a group of NULLs. - ( - DataType::Float64, - groups.is_empty() || plain(&input, *column)?.1, - ) + // NULL for a group of NULLs, as SQL `SUM` (and for a + // global one over no row: `with_emit_empty_global`). + (DataType::Float64, plain(&input, *column)?.1) } Reduction::Min(i) | Reduction::Max(i) => { let (t, nullable) = plain(&input, *i)?; if !ordered(t) { return Err(invalid("ordered aggregate input required")); } - (t.clone(), nullable || groups.is_empty()) + (t.clone(), nullable) } Reduction::StdDev { column, population } | Reduction::Variance { column, population } => { @@ -83,15 +81,14 @@ impl Operator { if !matches!(t, DataType::Int64 | DataType::Float64) { return Err(invalid("numeric aggregate input required")); } - // SQL: NULL for fewer than two values (sample) or none - // (population); a PromQL group always has a value. The - // IR's aggregate schema defines this nullability; this + // NULL for a group of NULLs and, as SQL's sample + // `STDDEV`/`VAR_SAMP`, for a group of one value (and for + // a global one over no row: `with_emit_empty_global`). + // PromQL's `stddev`/`stdvar` are population ones over + // groups that have a value, so never NULL. The IR's + // aggregate schema defines this nullability; this // derives it again to check the declared output. - ( - DataType::Float64, - !input.has_promql_series_identity() - && (!population || groups.is_empty() || nullable), - ) + (DataType::Float64, !population || nullable) } Reduction::PearsonCorr { left, right } => { for column in [left, right] { @@ -113,6 +110,7 @@ impl Operator { groups, measures: measures.into_iter().map(|(_, r)| r).collect(), filters: vec![], + emit_empty_global: false, }, inputs: vec![input], output: schema(fields), @@ -128,6 +126,7 @@ impl Operator { filters: slot, measures, groups, + .. } = &mut self.kind else { return Err(invalid("measure filters need an aggregate")); @@ -151,16 +150,7 @@ impl Operator { if filter.dtype(&self.inputs[0])?.0 != DataType::Bool { return Err(invalid("measure filter must be boolean")); } - if matches!( - measures[index], - Reduction::Sum(_) - | Reduction::Avg(_) - | Reduction::Min(_) - | Reduction::Max(_) - | Reduction::Percentile { .. } - | Reduction::StdDev { .. } - | Reduction::Variance { .. } - ) { + if measures[index].is_null_without_values() { output.fields[groups.len() + index].nullable = true; } } @@ -293,6 +283,22 @@ pub enum Reduction { /// PromQL `group`: 1 for every group. Group, } +impl Reduction { + /// Whether the measure is NULL for a group with no (matching) value, as + /// SQL `SELECT sum(x) FROM t WHERE false` is NULL while `count(*)` is 0. + pub(super) fn is_null_without_values(&self) -> bool { + matches!( + self, + Reduction::Sum(_) + | Reduction::Avg(_) + | Reduction::Min(_) + | Reduction::Max(_) + | Reduction::Percentile { .. } + | Reduction::StdDev { .. } + | Reduction::Variance { .. } + ) + } +} pub(super) fn execute<'a>( operator: &'a Operator, mut inputs: Vec>, @@ -333,22 +339,7 @@ pub(super) fn execute<'a>( results.len(), ) } - Kind::Aggregate { - groups, - measures, - filters, - } => { - columnar::aggregate( - input, - groups, - measures, - filters, - &operator.inputs[0], - &output, - &context, - ) - .await - } + Kind::Aggregate { .. } => columnar::aggregate(input, operator, &context).await, _ => unreachable!(), } }) diff --git a/crates/executor/src/operators/mod.rs b/crates/executor/src/operators/mod.rs index ded0132f9..8141ab4b0 100644 --- a/crates/executor/src/operators/mod.rs +++ b/crates/executor/src/operators/mod.rs @@ -156,6 +156,12 @@ enum Kind { /// Per-measure row filters (SQL `FILTER (WHERE …)`); empty when none. #[serde(default, skip_serializing_if = "Vec::is_empty")] filters: Vec>, + /// Whether an empty input still has the global group, as a SQL + /// `SELECT count(*) FROM t WHERE false` has one row (0); a PromQL + /// `count(v)` over an empty vector has no sample. Only an aggregate + /// with no groups; `false` (no output row) by default. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + emit_empty_global: bool, }, SemiJoin { keys: Vec<(usize, usize)>, @@ -179,6 +185,11 @@ enum Kind { /// Rows for which this is not true update no state; their group is kept. #[serde(default, skip_serializing_if = "Option::is_none")] filter: Option>, + /// Whether an empty input still has the global group, with an empty + /// state, as a global SQL aggregate has one row over no input rows. + /// Only a build with no groups; `false` (no output row) by default. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + emit_empty_global: bool, }, /// One keyed state per group whose items take a row weight, e.g. a CMS /// with heap (built by [`Operator::per_group_keyed_summary_build`]). @@ -352,6 +363,71 @@ impl Operator { pub fn schema(&self) -> SchemaRef { self.output.clone() } + + /// An aggregate, per-group build or merge into no groups follows SQL's + /// rule for an empty input when `emit_empty_global`: its one global + /// group still has a row, as `SELECT count(*) FROM t WHERE false` + /// answers 0 and `SELECT sum(x) FROM t WHERE false` NULL. Otherwise (the + /// default; `false` leaves the operator as it is) an empty input has no + /// row, as a PromQL `count(v)` over an empty vector has no sample. + /// + /// An `Aggregate`'s SUM, AVG, MIN, MAX, percentile, standard deviation + /// and variance measures become nullable (NULL over no row). A `PerGroupSummaryBuild` outputs an empty + /// state of its own θ, and a `SummaryMerge` one of `parameters`: the θ + /// of the states it merges (the exported build node's `parameters`), + /// `None` for an exact family and for the other two. + pub fn with_emit_empty_global( + mut self, + emit_empty_global: bool, + parameters: Option, + ) -> Result { + if !emit_empty_global { + return Ok(self); + } + match &mut self.kind { + Kind::Aggregate { + groups, + measures, + emit_empty_global, + .. + } if groups.is_empty() && parameters.is_none() => { + let mut output = (*self.output).clone(); + for (field, measure) in output.fields.iter_mut().zip(measures.iter()) { + if measure.is_null_without_values() { + field.nullable = true; + } + } + *emit_empty_global = true; + self.output = Arc::new(output); + } + Kind::PerGroupSummaryBuild { + groups, + emit_empty_global, + .. + } if groups.is_empty() && parameters.is_none() => *emit_empty_global = true, + Kind::SummaryMerge { + state, + groups, + emit_empty_global, + parameters: slot, + } if groups.is_empty() => { + // An empty state of this family and θ can be built. + crate::StateColumnUpdater::new( + &self.inputs[0].fields[*state].dtype, + parameters.as_ref(), + )?; + *emit_empty_global = true; + *slot = parameters; + } + _ => { + return Err(invalid( + "an empty global row needs an aggregate, build or merge into no groups \ + (only a merge takes the θ of its empty state)", + )) + } + } + Ok(self) + } } impl PhysicalOperator for Operator { fn requires_bounded_input(&self) -> bool { diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 7faab29cd..7e0e95cd6 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -277,6 +277,7 @@ impl Operator { time, groups, filter: None, + emit_empty_global: false, }, inputs: vec![input], output: schema(fields), @@ -308,36 +309,6 @@ impl Operator { output: schema(fields), }) } - /// A merge into no groups that outputs one empty state when it has no - /// input state: the global SQL aggregate a rollup of a relation's - /// grouped states to no keys stands for, whose one row is NULL over no - /// input rows. `parameters` is the merged states' θ, which the empty - /// state takes: the exported build node's `parameters` (`None` for an - /// exact family). - pub fn with_emit_empty_global( - mut self, - parameters: Option, - ) -> Result { - match &mut self.kind { - Kind::SummaryMerge { - state, - groups, - emit_empty_global, - parameters: slot, - } if groups.is_empty() => { - // An empty state of this family and θ can be built. - StateColumnUpdater::new(&self.inputs[0].fields[*state].dtype, parameters.as_ref())?; - *emit_empty_global = true; - *slot = parameters; - } - _ => { - return Err(invalid( - "an empty global state needs a merge into no groups", - )) - } - } - Ok(self) - } pub fn evaluation( input: SchemaRef, state: usize, @@ -459,6 +430,7 @@ pub(super) fn execute<'a>( time, groups, filter, + emit_empty_global, } => Ok(futures::stream::once(async move { build_per_group_summary( input, @@ -469,7 +441,7 @@ pub(super) fn execute<'a>( time: *time, groups, filter: filter.as_deref(), - emit_empty_global: !input_schema.has_promql_series_identity(), + emit_empty_global: *emit_empty_global, }, &input_schema, &output, @@ -613,7 +585,8 @@ struct PerGroupBuild<'a> { time: Option, groups: &'a [usize], filter: Option<&'a Expression>, - /// Whether an empty input still has the global group (SQL; not PromQL). + /// Whether an empty input still has the global group (SQL; not PromQL): + /// the kind's `emit_empty_global`. emit_empty_global: bool, } @@ -634,7 +607,8 @@ async fn build_per_group_summary( let mut updater = StateColumnUpdater::new(build.family, build.parameters)?; let mut work = Cooperative::new(context); let mut workspace = Workspace::new(context)?; - // PromQL aggregation of an empty vector produces no sample. + // A global SQL aggregate has one row over no input rows; a PromQL + // aggregation of an empty vector produces no sample. if build.groups.is_empty() && build.emit_empty_global { table.add_empty_group()?; updater.add_group()?; diff --git a/crates/executor/src/operators/unchecked.rs b/crates/executor/src/operators/unchecked.rs index 69c47e522..dae78bb3e 100644 --- a/crates/executor/src/operators/unchecked.rs +++ b/crates/executor/src/operators/unchecked.rs @@ -152,6 +152,7 @@ impl TryFrom for Operator { groups, measures, filters, + emit_empty_global, } => { if groups.len() + measures.len() != output.fields.len() { return Err(invalid("aggregate width mismatch")); @@ -159,6 +160,7 @@ impl TryFrom for Operator { let names = output.fields[groups.len()..].iter().map(|f| f.name.clone()); Operator::aggregate(input(0)?, groups, names.zip(measures).collect())? .with_measure_filters(filters)? + .with_emit_empty_global(emit_empty_global, None)? } Kind::SemiJoin { keys, @@ -190,6 +192,7 @@ impl TryFrom for Operator { time, groups, filter, + emit_empty_global, } => with_filter( Operator::per_group_summary_build( input(0)?, @@ -200,7 +203,8 @@ impl TryFrom for Operator { groups, )?, filter, - )?, + )? + .with_emit_empty_global(emit_empty_global, None)?, Kind::PerGroupKeyedSummaryBuild { family, parameters, @@ -246,16 +250,13 @@ impl TryFrom for Operator { emit_empty_global, parameters, } => { - let merge = Operator::summary_merge(input(0)?, state, groups)?; - if emit_empty_global { - merge.with_emit_empty_global(parameters)? - } else if parameters.is_some() { + if !emit_empty_global && parameters.is_some() { return Err(invalid( "only a merge that outputs an empty global state takes parameters θ", )); - } else { - merge } + Operator::summary_merge(input(0)?, state, groups)? + .with_emit_empty_global(emit_empty_global, parameters)? } Kind::Evaluation { state, query } => Operator::evaluation(input(0)?, state, query)?, } diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index f15c12e96..724a44f8b 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -10,6 +10,7 @@ use crate::{ values::{Batch, SchemaRef}, Error, }; +use planner_types::ir::operator::asap::evaluated_rows_kind; use planner_types::ir::operator::{ agg_has_exact_per_entity_kernel, AggIntent, GroupKeys, Reduction as PlannerReduction, }; @@ -28,7 +29,9 @@ use planner_types::summary_type::{ // `promql_rows` reads `OperatorNode` through `super::*` only in its // test-support helpers. #[cfg_attr(not(any(test, feature = "test-support")), allow(unused_imports))] -use planner_types::ir::{ASAPOp, NonASAPOp, OperatorNode, ScalarExpr, TimeRangeKind}; +use planner_types::ir::{ + ASAPOp, NonASAPOp, OperatorNode, OperatorResultKind, ScalarExpr, TimeRangeKind, +}; mod logical; use std::{ collections::{BTreeMap, BTreeSet}, @@ -256,6 +259,20 @@ fn compile_internal( .iter() .map(|id| Arc::new(nodes[id].output_schema.clone())) .collect::>(); + // Whether an aggregate, summary build or merge reads a SQL + // relation's rows (or states built from them), which gives it + // SQL's rules for an empty input and a quantile; no other + // operator reads it. + let over_relation = matches!( + node.payload, + Payload::NonASAP(NonASAPOp::Aggregate { .. }) + | Payload::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryMerge { .. } + ) + ) && input_rows_kind(&restored[&id]) + == OperatorResultKind::Relation; if matches!(node.payload, Payload::ASAP(ASAPOp::SummaryMerge { .. })) && inputs.len() > 1 { @@ -663,6 +680,7 @@ fn compile_internal( update, reduction, window_end, + over_relation, )?, }; let build = match filter { @@ -812,7 +830,7 @@ fn compile_internal( } if let Payload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) = &node.payload { // Exact counts evaluate to Int64; PromQL declares a Float64 sample. - let evaluation = bind_operation(node, &schemas) + let evaluation = bind_operation(node, &schemas, over_relation) .map_err(|error| invalid(format!("node {id}: {error}")))?; let actual = evaluation.schema(); let converted = actual.fields.iter().zip(&output.fields).position(|(a, d)| { @@ -861,7 +879,7 @@ fn compile_internal( // state covers this evaluation's window, so merge by // series identity and attach the run scope's timestamp, // as a per-series `SummaryAgg` does. - let merged = bind_operation(node, &schemas)?; + let merged = bind_operation(node, &schemas, over_relation)?; let compact = merged.schema(); physical_dag.add(helper_id(id, 1)?, inputs, merged)?; physical_dag.add( @@ -872,23 +890,6 @@ fn compile_internal( continue; } } - // A rollup of a relation's states to no keys is a global SQL - // aggregate: one row, an empty state, over no input rows. - if let Payload::ASAP(ASAPOp::SummaryMerge { - group_by: Some(keys), - .. - }) = &node.payload - { - if keys.is_empty() && built_from_relation(&restored[&id]) { - // The merge carries the θ of the states it reads. - let parameters = node.parameters.clone(); - let operator = compile_node(node, &schemas) - .and_then(|merge| merge.with_emit_empty_global(parameters)) - .map_err(|error| invalid(format!("node {id}: {error}")))?; - physical_dag.add(id, inputs, operator)?; - continue; - } - } // A window over a table or a time series (and a tumbling window's // shift under it) runs natively on the time column. if let Some(operator) = time_window_operator(id, &graph, &schemas) @@ -897,7 +898,7 @@ fn compile_internal( physical_dag.add(id, inputs, operator.with_output_schema(output)?)?; continue; } - let mut operator = compile_node(node, &schemas) + let mut operator = compile_node(node, &schemas, over_relation) .map_err(|error| invalid(format!("node {id}: {error}")))?; if operator.is_counter_evaluation() { if let Some((offset, lookback)) = counter_window(id, &nodes, &dependencies)? { @@ -916,19 +917,19 @@ fn compile_internal( Ok(physical_dag) } -/// Whether every summary state `state` merges was built from a relation's -/// rows (a SQL table's), where a global aggregate has a row over no input -/// row; not from a PromQL vector's, where it has none. -fn built_from_relation(state: &std::rc::Rc) -> bool { - match state.asap() { - Some(ASAPOp::SummaryMerge { children, .. }) => children.iter().all(built_from_relation), - Some( - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => child.result_kind == planner_types::ir::OperatorResultKind::Relation, - // A filter keeping whole states reads them from their build. - _ => planner_types::ir::operator::asap::selected_from(state) - .is_some_and(|states| built_from_relation(&states)), - } +/// The result kind of the rows `node` reads: its (first) input's, or, when +/// that input is summary states, the kind of the rows they were built from +/// ([`evaluated_rows_kind`]). It decides an aggregate's rule for an empty +/// input: a global aggregate over a SQL table's `Relation` has one row over +/// no input rows (`SELECT count(*) FROM t WHERE false` is 0), over a PromQL +/// vector it has none (`count(sum by (job) (m >= 1000))` is empty when no +/// value reaches 1000), whether or not its input still has a series +/// identity column. +fn input_rows_kind(node: &OperatorNode) -> OperatorResultKind { + node.operator + .children() + .first() + .map_or(node.result_kind, |input| evaluated_rows_kind(*input)) } // Temporal summary evaluations produce PromQL vectors, whose range functions drop @@ -973,14 +974,31 @@ pub(crate) fn require_sound_merge(input: &SchemaRef) -> Result<(), Error> { /// that lowers to one operator. A node that lowers to several (an exact /// top-k, `count_values`, or a `without` grouping over a series identity) /// is lowered only by [`compile`]; here it is refused. -pub fn compile_node(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { +/// `over_relation`: whether the node reads a SQL relation's rows (for +/// summary states, whether they were built from one). Then an aggregate, +/// build or merge into no groups has SQL's one row over no input rows, and +/// a quantile is SQL's percentile. Otherwise (a PromQL vector, a range +/// vector, or states built from either) it has no row and PromQL's +/// quantile. +pub fn compile_node( + node: &PhysicalASAPDAGNode, + inputs: &[SchemaRef], + over_relation: bool, +) -> Result { for schema in inputs { crate::values::validate_schema(schema)?; } - bind_operation(node, inputs)?.with_output_schema(Arc::new(node.output_schema.clone())) + bind_operation(node, inputs, over_relation)? + .with_output_schema(Arc::new(node.output_schema.clone())) } -fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { +/// `over_relation`: whether the node reads a SQL relation's rows (or +/// states built from them), not a PromQL vector's. +fn bind_operation( + node: &PhysicalASAPDAGNode, + inputs: &[SchemaRef], + over_relation: bool, +) -> Result { if let Payload::NonASAP(NonASAPOp::BinaryOp { operator, return_bool, @@ -1259,7 +1277,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { Reduction::Quantile { @@ -1270,9 +1288,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result - { + AggIntent::Quantile { col, q, .. } if over_relation => { Reduction::Percentile { column: column(*col)?, q: *q, @@ -1287,13 +1303,16 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result>()?; - Operator::aggregate(input.clone(), groups(input, keys)?, measures)? + let groups = groups(input, keys)?; + let global = groups.is_empty(); + Operator::aggregate(input.clone(), groups, measures)? .with_measure_filters( filters .iter() .map(|f| f.as_ref().map(|f| expression(&f.0, input)).transpose()) .collect::>()?, - ) + )? + .with_emit_empty_global(global && over_relation, None) } _ => Err(invalid("value operation has no native implementation")), }, @@ -1317,6 +1336,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result build.with_row_filter(expression(&filter.0, input)?), @@ -1343,7 +1363,12 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { if let SketchStatistic::TopK { k } = query { @@ -1381,7 +1406,9 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result, + over_relation: bool, ) -> Result { let groups = |input: &SchemaRef, keys: &GroupKeys| -> Result, Error> { let mut columns = groups(input, keys)?; columns.extend(window_end); Ok(columns) }; + // A state per group of `groups` over `value`. + let per_group = |value: usize, groups: Vec| -> Result { + let global = groups.is_empty(); + Operator::per_group_summary_build( + input.clone(), + family.clone(), + parameters.cloned(), + value, + input.time_index, + groups, + )? + .with_emit_empty_global(global && over_relation, None) + }; if matches!( family, FieldDataType::Sketch(planner_types::summary_type::SketchAlgorithm::HydraCms) @@ -1431,14 +1472,7 @@ fn summary_agg_build( "summary construction requires explicit grouping columns", )); }; - return Operator::per_group_summary_build( - input.clone(), - family.clone(), - parameters.cloned(), - named_column(input, item)?, - input.time_index, - groups(input, keys)?, - ); + return per_group(named_column(input, item)?, groups(input, keys)?); } if let Some(item) = &update.item { let PlannerReduction::Reduce(keys) = reduction else { @@ -1483,14 +1517,7 @@ fn summary_agg_build( "summary construction requires explicit grouping columns", )); }; - Operator::per_group_summary_build( - input.clone(), - family.clone(), - parameters.cloned(), - named_column(input, column)?, - input.time_index, - groups(input, keys)?, - ) + per_group(named_column(input, column)?, groups(input, keys)?) } /// What each row adds in a keyed build: a projected weight column, or a /// positive integer constant (1 for a row count) read without a column. diff --git a/crates/executor/src/physical_planner/tests.rs b/crates/executor/src/physical_planner/tests.rs index fcf065b20..b9514250f 100644 --- a/crates/executor/src/physical_planner/tests.rs +++ b/crates/executor/src/physical_planner/tests.rs @@ -75,3 +75,70 @@ fn a_stored_time_shift_over_a_chain_reads_the_chain_unbounded() { assert_eq!(window_read_above(0, &graph), None); } } + +/// A global merge (`group_by: Some([])`) of summary states built from a SQL +/// table's rows reads a `Relation`, so it keeps SQL's one row over no input +/// rows, also when another summary operator sits between the build and the +/// merge: `SummaryMerge([]) ← SummarySubtract ← SummaryAgg ← Scan(t)` and +/// the same with `SummaryJoin`. Over a PromQL series scan the same tree +/// reads an instant vector (no row over an empty vector). +#[test] +fn a_global_merge_reads_the_rows_its_states_were_built_from() { + use planner_types::ir::operator::arguments::Source; + use planner_types::ir::operator::Reduction as IrReduction; + use planner_types::ir::schema::Schema; + use planner_types::ir::Operator as IrOperator; + use planner_types::summary_type::{ExactKind, ExactParams, SummaryUpdate}; + use std::rc::Rc; + let node = |operator| Rc::new(OperatorNode::with_schema(operator, Schema::default())); + for (source, expected) in [ + ( + Source::Table { + table_ref: "t".into(), + }, + OperatorResultKind::Relation, + ), + ( + Source::TimeSeries { metric: "m".into() }, + OperatorResultKind::InstantVector, + ), + ] { + let scan = node(IrOperator::NonASAP(NonASAPOp::Scan { + source, + predicates: vec![], + schema: Schema::default(), + })); + let build = node(IrOperator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: IrReduction::PerEntity, + filter: None, + })); + let subtract = node(IrOperator::ASAP(ASAPOp::SummarySubtract { + left: build.clone(), + right: build.clone(), + })); + let join = node(IrOperator::ASAP(ASAPOp::SummaryJoin { + outer: build.clone(), + inner: build.clone(), + key: 0, + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + })); + for states in [subtract, join] { + let merge = OperatorNode::with_schema( + IrOperator::ASAP(ASAPOp::SummaryMerge { + children: vec![states.clone()], + group_by: Some(vec![]), + }), + Schema::default(), + ); + assert_eq!( + input_rows_kind(&merge), + expected, + "{}", + states.operator.kind_name() + ); + } + } +} diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index bc889dd24..5b72f4a04 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -168,7 +168,12 @@ fn try_run( batches: Vec, ) -> Result, String> { let input = input_schema(); - let build = compile_node(&summary_agg(update, filter), std::slice::from_ref(&input)).unwrap(); + let build = compile_node( + &summary_agg(update, filter), + std::slice::from_ref(&input), + true, + ) + .unwrap(); let state = build.schema(); let mut operators = BTreeMap::new(); let mut states = vec![]; @@ -199,6 +204,7 @@ fn try_run( (*state).clone(), ), std::slice::from_ref(&state), + true, ) .unwrap(); operators.insert(201, (vec![200], merge)); @@ -207,6 +213,7 @@ fn try_run( let read = compile_node( &estimate(3, point("checkout"), DataType::Float64), std::slice::from_ref(&state), + true, ) .unwrap(); operators.insert(300, (vec![merged], read)); @@ -327,9 +334,13 @@ fn positive_constant_weight_adds_the_constant_per_row() { assert_eq!(doubled[job], 2.0 * unit[job], "{job}"); } for weight in [0.0, -2.0, 2.5, f64::INFINITY] { - let error = compile_node(&summary_agg(constant(weight), None), &[input_schema()]) - .err() - .unwrap_or_else(|| panic!("constant weight {weight} accepted")); + let error = compile_node( + &summary_agg(constant(weight), None), + &[input_schema()], + true, + ) + .err() + .unwrap_or_else(|| panic!("constant weight {weight} accepted")); assert!(error.to_string().contains("HydraCms"), "{weight}: {error}"); } } @@ -349,7 +360,7 @@ fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { (constant, serde_json::json!({"Constant": 2})), (column, serde_json::json!({"Column": 2})), ] { - let build = compile_node(&summary_agg(update, None), &[input_schema()]).unwrap(); + let build = compile_node(&summary_agg(update, None), &[input_schema()], true).unwrap(); let wire = serde_json::to_value(&build).unwrap(); let fields = &wire["kind"]["AllGroupsSummaryBuild"]; assert_eq!(fields["row_weight"], expected, "{wire}"); @@ -399,7 +410,7 @@ fn constant_weight_above_i32_is_refused_at_validation() { error.contains("HydraCms") && error.contains("i32::MAX"), "{error}" ); - let error = compile_node(&summary_agg(update, None), &[input_schema()]) + let error = compile_node(&summary_agg(update, None), &[input_schema()], true) .err() .expect("constant above i32::MAX compiled"); assert!( @@ -431,14 +442,14 @@ fn invalid_weight_values_fail_the_build() { // rather than computed from sketchlib internals. #[test] fn group_total_is_rejected_until_sketchlib_offers_it() { - let state = compile_node(&summary_agg(unit_count(), None), &[input_schema()]) + let state = compile_node(&summary_agg(unit_count(), None), &[input_schema()], true) .unwrap() .schema(); let total = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, }; - let error = compile_node(&estimate(3, total, DataType::Int64), &[state]) + let error = compile_node(&estimate(3, total, DataType::Int64), &[state], true) .err() .expect("group total"); assert!( diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index fe1873d52..92a7300b5 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -175,7 +175,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { selected_physical_candidate: None, view_coverage: None, }; - let build = compile_node(&node, std::slice::from_ref(&input)).unwrap(); + let build = compile_node(&node, std::slice::from_ref(&input), true).unwrap(); let wire = serde_json::to_value(&build).unwrap(); assert_eq!( wire["kind"]["PerGroupKeyedSummaryBuild"]["row_weight"], diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 5dbdc04e1..95013aa3d 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -305,7 +305,10 @@ fn exact_integer_and_empty_extrema() { .add( 1, vec![0], - Operator::aggregate(schema, vec![], vec![("min".into(), Reduction::Min(0))]).unwrap(), + Operator::aggregate(schema, vec![], vec![("min".into(), Reduction::Min(0))]) + .unwrap() + .with_emit_empty_global(true, None) + .unwrap(), Lifetime::None, ) .unwrap(); @@ -646,7 +649,8 @@ fn bind_physical_asap_dag_before_execution() { assert!(bind(&scanned, data(), &[1]).is_err()); } -// A completed empty population has an exact zero count, with integer output. +// A completed empty population of a global SQL count has an exact zero +// count, with integer output. #[test] fn empty_exact_count_is_an_integer_state_evaluation() { let input = schema(&[("value", DataType::Float64, false)]); @@ -658,6 +662,8 @@ fn empty_exact_count_is_an_integer_state_evaluation() { None, vec![], ) + .unwrap() + .with_emit_empty_global(true, None) .unwrap(); let read = Operator::evaluation( build.schema(), @@ -1301,7 +1307,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { let join = if certified { Operator::certified_semi_join(schema.clone(), schema.clone(), vec![(0, 0)]).unwrap() } else { - compile_node(&node, &[schema.clone(), schema.clone()]).unwrap() + compile_node(&node, &[schema.clone(), schema.clone()], true).unwrap() }; // Each side is a leaf holding its keys. let plan = |left: Vec<&str>, right: Vec<&str>| { @@ -1405,7 +1411,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { selected_physical_candidate: None, view_coverage: None, }; - let binary = compile_node(&node, &[input.clone(), input.clone()]).unwrap(); + let binary = compile_node(&node, &[input.clone(), input.clone()], false).unwrap(); for (right, expected) in [ (vec![("b", 2, 3.), ("a", 1, 2.)], Some(vec![8., 17.])), (vec![("b", 2, 3.)], None), diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index f6958dc53..19d2d0715 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -729,7 +729,8 @@ fn grouping_canonicalizes_null_nan_and_signed_zero() { assert!(result.iter().all(|r| matches!(r[1], Value::Int64(2)))); } -// Global empty input yields one aggregate row; grouped empty input yields none. +// Global empty input yields one aggregate row under SQL's rule +// (`with_emit_empty_global`); grouped empty input yields none. #[test] fn aggregate_empty_and_all_null_follow_asap_contract() { let input = schema(&[("v", DataType::Int64, true)]); @@ -748,6 +749,8 @@ fn aggregate_empty_and_all_null_follow_asap_contract() { ("max".into(), Reduction::Max(0)), ], ) + .unwrap() + .with_emit_empty_global(true, None) .unwrap(); let result = unary(input.clone(), batches, op); assert_eq!(result.len(), 1); @@ -823,7 +826,7 @@ fn global_extrema_bind_with_planner_derived_schema() { selected_physical_candidate: None, view_coverage: None, }; - let operator = compile_node(&node, std::slice::from_ref(&input)) + let operator = compile_node(&node, std::slice::from_ref(&input), true) .expect("global extremum should bind to its Planner schema"); assert!(operator.schema().fields[0].nullable); let empty = unary(input.clone(), vec![], operator.clone()); @@ -1301,7 +1304,8 @@ fn zero_column_output_obeys_memory_limit() { assert_eq!(run.retained_bytes(), 0); } -// Empty exact-state finalization must preserve ordinary global MIN/MAX null semantics. +// Empty exact-state finalization must preserve ordinary global SQL MIN/MAX +// null semantics. #[test] fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { use planner_types::summary_type::{ExactKind, ExactParams}; @@ -1319,6 +1323,8 @@ fn empty_exact_summary_extrema_agree_with_ordinary_aggregation() { None, vec![], ) + .unwrap() + .with_emit_empty_global(true, None) .unwrap(); let state = build.schema(); let mut dag = PhysicalDAG::default(); @@ -1465,7 +1471,7 @@ fn measure_filters_keep_groups_without_matching_rows() { ("all", plain(DataType::Int64), false), ], ); - let operator = round_trip(compile_node(&node, std::slice::from_ref(&input)).unwrap()); + let operator = round_trip(compile_node(&node, std::slice::from_ref(&input), true).unwrap()); assert_eq!( printed(&unary(input, vec![rows], operator)), ["a 1 1.0 2", "b 0 NULL 1", "c 1 4.0 1"] @@ -1493,7 +1499,7 @@ fn filtered_sum_declared_non_null_is_rejected() { ("s", FieldDataType::Plain(DataType::Float64), false), ], ); - assert!(compile_node(&node, &[input]).is_err()); + assert!(compile_node(&node, &[input], true).is_err()); } /// A filtered `SummaryAgg` updates its state only from kept rows but keeps @@ -1544,7 +1550,7 @@ fn filtered_summary_build_keeps_empty_groups() { ], ); asap_executor::physical_planner::test_support::set_build_parameters([&mut build], None); - let build = round_trip(compile_node(&build, std::slice::from_ref(&input)).unwrap()); + let build = round_trip(compile_node(&build, std::slice::from_ref(&input), true).unwrap()); let read = planner_node( PhysicalASAPOperatorPayload::ASAP(read), &[ @@ -1552,7 +1558,7 @@ fn filtered_summary_build_keeps_empty_groups() { ("value", FieldDataType::Plain(result), nullable), ], ); - let read = compile_node(&read, &[build.schema()]).unwrap(); + let read = compile_node(&read, &[build.schema()], true).unwrap(); let mut dag = PhysicalDAG::default(); dag.add( 0, @@ -1598,7 +1604,7 @@ fn filtered_summary_build_is_rejected_at_ingestion_time() { ], ); node.output_state = ExecutionDataState::INGESTION_SUMMARY; - compile_node(&node, std::slice::from_ref(&input)) + compile_node(&node, std::slice::from_ref(&input), true) }; assert!(build(None).is_ok()); let error = build(keep()).err().expect("filtered ingestion-time build"); @@ -1665,6 +1671,88 @@ fn exact_aggregate_node( } } +// A global aggregate follows the result kind of the rows it reads, not +// whether they carry a series identity column. Over `[job, v]`, the output +// of `max by (job) (m)` (no identity column), PromQL's `sum(…)`, `avg`, +// `min`, `max`, `quantile(0.5, …)`, `stddev` and `stdvar` are non-null and +// answer no sample over an empty vector; over the same columns as a SQL +// relation they are nullable and answer one NULL row over no row. Over jobs +// a (1) and b (3) both answer 4, 2, 1, 3, 2, 1 and 1 (mean 2, population +// variance ((1 - 2)² + (3 - 2)²) / 2 = 1). +#[test] +fn a_global_aggregate_follows_its_input_rows_kind() { + use asap_executor::physical_planner::compile_node; + use planner_types::ir::operator::AggIntent; + + let input = schema(&[ + ("job", DataType::Utf8, false), + ("v", DataType::Float64, false), + ]); + let rows = vec![ + vec![Value::Utf8("a".into()), Value::Float64(1.0)], + vec![Value::Utf8("b".into()), Value::Float64(3.0)], + ]; + for (measure, expected) in [ + (AggIntent::Sum { col: Some(1) }, 4.0), + (AggIntent::Avg { col: Some(1) }, 2.0), + (AggIntent::Min { col: Some(1) }, 1.0), + (AggIntent::Max { col: Some(1) }, 3.0), + ( + AggIntent::Quantile { + col: Some(1), + q: 0.5, + }, + 2.0, + ), + ( + AggIntent::StdDev { + col: Some(1), + population: true, + }, + 1.0, + ), + ( + AggIntent::Variance { + col: Some(1), + population: true, + }, + 1.0, + ), + ] { + let mut node = exact_aggregate_node(measure.clone(), "value", DataType::Float64); + // A PromQL vector: non-null, no sample over an empty vector. + let promql = compile_node(&node, std::slice::from_ref(&input), false) + .unwrap_or_else(|error| panic!("{measure:?}: {error}")); + assert!( + unary(input.clone(), vec![], promql.clone()).is_empty(), + "{measure:?}" + ); + let answer = unary(input.clone(), vec![rows.clone()], promql); + assert!( + matches!(answer.as_slice(), [row] if matches!(row[0], Value::Float64(v) if v == expected)), + "{measure:?}: {answer:?}" + ); + // A SQL relation: NULL over no row, so a non-null output is refused. + assert!(compile_node(&node, std::slice::from_ref(&input), true).is_err()); + node.output_schema.fields[0].nullable = true; + // The rule survives serialization. + let sql = round_trip( + compile_node(&node, std::slice::from_ref(&input), true) + .unwrap_or_else(|error| panic!("{measure:?}: {error}")), + ); + let empty = unary(input.clone(), vec![], sql.clone()); + assert!( + matches!(empty.as_slice(), [row] if matches!(row[0], Value::Null)), + "{measure:?}: {empty:?}" + ); + let answer = unary(input.clone(), vec![rows.clone()], sql); + assert!( + matches!(answer.as_slice(), [row] if matches!(row[0], Value::Float64(v) if v == expected)), + "{measure:?}: {answer:?}" + ); + } +} + // Exact frequency intents bind to native reducers over typed keys, without a // sketch or numeric key conversion; empty and all-NULL input give 0. #[test] @@ -1704,7 +1792,7 @@ fn exact_frequency_intents_execute_typed_keys_and_empty_input() { ), ] { let node = exact_aggregate_node(measure, name, DataType::Float64); - let operator = compile_node(&node, std::slice::from_ref(&input)) + let operator = compile_node(&node, std::slice::from_ref(&input), true) .expect("exact frequency intent binds"); let rows = values .iter() @@ -1809,8 +1897,8 @@ fn exact_cardinality_binds_and_executes_typed_tuples() { for (cols, expected) in [(vec![0], 2), (vec![0, 1], 3)] { let node = exact_aggregate_node(AggIntent::Cardinality { cols }, "distinct", DataType::Int64); - let operator = - compile_node(&node, std::slice::from_ref(&input)).expect("exact distinct intent binds"); + let operator = compile_node(&node, std::slice::from_ref(&input), true) + .expect("exact distinct intent binds"); let rows = vec![ vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], vec![Value::Int64(9_007_199_254_740_992), Value::Utf8("a".into())], @@ -1921,7 +2009,11 @@ fn sql_variance_and_stddev_follow_their_definitions() { ] }; let run = |values: &[Option]| { - let op = Operator::aggregate(input.clone(), vec![], measures()).unwrap(); + // A global SQL aggregate: one row also over no input row. + let op = Operator::aggregate(input.clone(), vec![], measures()) + .unwrap() + .with_emit_empty_global(true, None) + .unwrap(); let rows = unary(input.clone(), vec![floats(values)], op); (0..4).map(|c| float_at(&rows, 0, c)).collect::>() }; @@ -2030,6 +2122,9 @@ fn correlation_follows_datafusion() { vec![], vec![("corr".into(), Reduction::PearsonCorr { left: 0, right: 1 })], ) + .unwrap() + // A global SQL aggregate: one row also over no input row. + .with_emit_empty_global(true, None) .unwrap(); let rows = pairs .iter() diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 0643bd71f..8954eeee9 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -86,7 +86,7 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { selected_physical_candidate: None, view_coverage: None, }; - compile_node(&node, &[schema.clone(), schema]).unwrap() + compile_node(&node, &[schema.clone(), schema], false).unwrap() } /// `binary` (node 2) over the sources `left` (node 0) and `right` (node 1), /// bound to a store with no data: the sources hold their rows. diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 1866ac810..63e9c8237 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -199,7 +199,7 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() selected_physical_candidate: None, view_coverage: None, }; - let build = compile_node(&node, std::slice::from_ref(&input))?; + let build = compile_node(&node, std::slice::from_ref(&input), true)?; let batch = Batch::try_new( input, ["a", "a", "b", "b"] diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 32804d4be..4fba7b99c 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use crate::time_series; use asap_executor::runtime::Scope; -use asap_executor::values::Batch; +use asap_executor::values::{Batch, Value}; use asap_planner::{ plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, OperatorKind, PlanningModels, PlanningWorkload, SelectedPlanReport, UserInput, @@ -788,3 +788,57 @@ async fn counter_rates_over_a_shared_query_time_chain_answer_as_installed() { } } } + +// ── Global aggregates over an aggregate's output ───────────────────────────── + +/// A PromQL aggregate with no grouping keys over another aggregate's output +/// is still a PromQL aggregate: over an empty vector it answers no sample, +/// as Prometheus does, not the one row (0) a SQL `SELECT count(*)` has over +/// no rows. Every value of `m` lies above 0 and below 1,000 ([`m_samples`]), +/// so `count(sum by (job) (m >= 1000))` counts an empty vector while +/// `count(sum by (job) (m >= 0))` counts the 2 jobs, and +/// `count(count by (x) (m))` counts one group, as no series has `x`. +/// `stddev` and `stdvar` follow the same rule: none over the empty vector, +/// and 0 over that one group. With no sample at all every query is empty. +/// Each query is planned on its own, exactly. +#[tokio::test] +async fn a_global_promql_count_over_an_empty_aggregate_answers_no_sample() { + // The query's answer values over `samples` (an answer row has no labels). + let values = |plan: &InstallablePlan, samples: &[RawDataSample]| -> Vec { + let query_time = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + run( + &plan.dag, + time_series_tables(&plan.dag, samples), + &[], + &query_roots(plan), + query_time, + ) + .concat() + .iter() + .flat_map(Batch::to_rows) + .map(|row| match row.as_slice() { + [Value::Int64(count)] => *count as f64, + [Value::Float64(value)] => *value, + other => panic!("an answer row {other:?}"), + }) + .collect() + }; + for (query, with_samples) in [ + ("count(sum by (job) (m >= 1000))", vec![]), + ("count(count by (x) (m))", vec![1.0]), + ("count(sum by (job) (m >= 0))", vec![2.0]), + ("stddev(sum by (job) (m >= 1000))", vec![]), + ("stdvar(sum by (job) (m >= 1000))", vec![]), + ("stdvar(count by (x) (m))", vec![0.0]), + ] { + let plan = plan_every_minute(&[(query, "")]).await; + assert_eq!( + values(&plan, &[]), + Vec::::new(), + "{query} with no sample" + ); + assert_eq!(values(&plan, &m_samples()), with_samples, "{query}"); + } +} diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 309a83ace..1d4d5e7e9 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -147,6 +147,7 @@ fn shown(value: &Value) -> String { match value { Value::Int64(n) => format!("Int64({n})"), Value::Float64(x) => format!("Float64({x})"), + Value::Null => "NULL".to_string(), other => format!("{other:?}"), } } @@ -439,3 +440,48 @@ async fn a_window_equal_to_its_interval_is_built_at_ingestion_time_from_table_ro .collect(); assert_eq!(by_key(run_data_plane(&plan).remove(0)), expected); } + +/// A global SQL aggregate has one row even over no input row, as SQL +/// requires: over the last 5 minutes with `WHERE false`, `COUNT(*)` is 0 +/// and `SUM`, `STDDEV_POP` and `VAR_SAMP` are NULL. Over the last 5 minutes of [`rows`] (`latency` = +/// 900, 901, …, 1199) they are 300 and 314,850. +#[tokio::test] +async fn a_global_sql_aggregate_over_no_row_answers_one_row() { + let last_5_minutes = "FROM http WHERE ts >= now() - INTERVAL '5 minutes'"; + let queries = [ + format!("SELECT count(*) {last_5_minutes} AND false"), + format!("SELECT sum(latency) {last_5_minutes} AND false"), + format!("SELECT stddev_pop(latency) {last_5_minutes} AND false"), + format!("SELECT var_samp(latency) {last_5_minutes} AND false"), + format!("SELECT count(*) {last_5_minutes}"), + format!("SELECT sum(latency) {last_5_minutes}"), + ]; + let queries = queries.each_ref().map(String::as_str); + let plan = plan_for( + &queries, + EXACT, + MINUTE_MS as u64, + &DeploymentCapabilities::default(), + ) + .await; + let answers: Vec>> = run_data_plane(&plan) + .into_iter() + .map(|rows| { + rows.into_iter() + .map(|row| row.iter().map(shown).collect()) + .collect() + }) + .collect(); + assert_eq!( + answers, + [ + "Int64(0)", + "NULL", + "NULL", + "NULL", + "Int64(300)", + "Float64(314850)", + ] + .map(|value| vec![vec![value.to_string()]]) + ); +} diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 44279f9de..4376a0553 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -961,6 +961,19 @@ pub fn summary_type_of_operator(operator: &Operator) -> Option<&FieldDataT } } +/// The category of the rows `node` stands for once evaluated: its own result +/// kind, or, for summary states, the kind of the rows they were built from +/// ([`source_kind`]). Sum states built over a SQL table `t` and then +/// subtracted or merged stand for a `Relation`, so a global rollup of them +/// has SQL's one row over no input rows; built over a PromQL series `m` they +/// stand for an `InstantVector`, so it has no sample over an empty vector. +pub fn evaluated_rows_kind(node: &I) -> OperatorResultKind { + match node.result_kind() { + OperatorResultKind::State => source_kind(node), + kind => kind, + } +} + /// The category of the values a evaluation of `state` produces: the category /// of the relational input the state was built from. fn source_kind(node: &I) -> OperatorResultKind {