From 3cabdc3b5f2880eca097ee1f8d2251fd3f125bc0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:34:13 +0000 Subject: [PATCH 01/12] feat(stage0, types): memo hash-consing, group properties (column_sign, depends_on_evaluation_time, aggregate_coverage), @ end()/@ start() lifted above their subtree (#1207 PR 2) - Memo: hash-consing (find_or_add_group by expression hash), candidate storage (add_candidate), add_query(MemoQuery); Memo::new(declared column signs), private declared_column_signs read by a Scan's column_sign. - Stage 0: memo_of_queries(roots, accuracy_targets, data) adds each query with its accuracy target and n_q (values_per_result over series_read), and collects the declared signs of the scanned sources (a counter's value is non-negative without a declaration). - GroupProperties::column_sign derived per operator (concepts doc, "Value sign"); depends_on_evaluation_time: a TimeRange or time()/now() makes a result depend on the evaluation time, @ T pins it; reads_only_time_series: a Scan of a time series, or every input group reading only time series (OperatorInput::reads_only_time_series computes the same rule over node trees; the recursive walks in topk.rs and Pass 1 are deleted). - aggregate_coverage for summary states, chains, TimeRange/TimeShift over a chain, merges and exact Aggregates; timestamp constraints are IntervalConstraintLimit intervals (t - o - W, t - o]; SQL timestamp literals and identity casts fold so absolute filters lift. - PromQL: @ end() / @ start() are lowered as TimeShift { at } above the selector or subquery they anchor, which is lowered without them, so it is the group of the same expression without @; coverage stops at that TimeShift (it is the definition's input group). --- crates/frontend-common/src/canonicalize.rs | 2 +- crates/frontend-common/src/resolve.rs | 20 +- crates/frontend-promql/src/lib.rs | 47 +- crates/frontend-promql/src/promql.rs | 94 ++- .../tests/promql_conformance.rs | 33 +- crates/frontend-sql/src/sql/expr.rs | 21 +- crates/frontend-sql/src/sql/types.rs | 7 + crates/frontend-sql/tests/sql_lowering.rs | 40 +- crates/stage0-memo/src/lib.rs | 648 ++++++++++++++++- .../src/computation_targets.rs | 13 +- crates/types/src/ir/operator/arguments.rs | 10 + crates/types/src/ir/operator/input.rs | 31 + crates/types/src/ir/operator/mod.rs | 6 +- .../src/ir/properties/summary_coverage.rs | 317 ++++++-- crates/types/src/ir/schema/series_identity.rs | 2 +- crates/types/src/ir/schema/topk.rs | 13 +- crates/types/src/stages/logical/candidate.rs | 2 +- .../src/stages/logical/column_properties.rs | 449 ++++++++++++ crates/types/src/stages/logical/group.rs | 89 ++- .../src/stages/logical/memo/hash_consing.rs | 74 ++ crates/types/src/stages/logical/memo/mod.rs | 177 ++++- crates/types/src/stages/logical/memo/tests.rs | 307 ++++++++ crates/types/src/stages/logical/mod.rs | 1 + .../src/stages/physical/memo_candidates.rs | 4 +- crates/types/tests/main.rs | 2 + crates/types/tests/memo_group_properties.rs | 491 +++++++++++++ crates/types/tests/memo_summary_coverage.rs | 681 ++++++++++++++++++ .../proposals/asap-primitive-schema.md | 244 ++++--- 28 files changed, 3539 insertions(+), 286 deletions(-) create mode 100644 crates/types/src/stages/logical/column_properties.rs create mode 100644 crates/types/src/stages/logical/memo/hash_consing.rs create mode 100644 crates/types/src/stages/logical/memo/tests.rs create mode 100644 crates/types/tests/memo_group_properties.rs create mode 100644 crates/types/tests/memo_summary_coverage.rs diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 8701574f3..6123568cb 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_stage0_memo`) can match +//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index ea3d3b442..79010baa4 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -487,11 +487,21 @@ fn resolve_expr_in( S::Not(e) => ScalarExpr::Not(bx(e)?), S::IsNull(e) => ScalarExpr::IsNull(bx(e)?), S::IsNotNull(e) => ScalarExpr::IsNotNull(bx(e)?), - S::Cast { expr, to, try_cast } => ScalarExpr::Cast { - expr: bx(expr)?, - to: to.clone(), - try_cast: *try_cast, - }, + // A cast to the type its operand already has is the operand: SQL's + // `ts >= '2026-11-01'` casts the Timestamp column `ts` to DataFusion's + // nanosecond timestamp, which is the same IR type. + S::Cast { expr, to, try_cast } => { + let operand = bx(expr)?; + if matches!(operand.scalar_type(schema), Ok((dtype, _)) if dtype == *to) { + *operand + } else { + ScalarExpr::Cast { + expr: operand, + to: to.clone(), + try_cast: *try_cast, + } + } + } S::InList { expr, list, diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index fa90b507d..f85eaaf4d 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -117,11 +117,12 @@ mod tests { use std::rc::Rc; use std::time::Duration; + use asap_types::ir::operator::arguments::{AtModifier, TimeShift}; use asap_types::ir::operator::Source; - use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; + use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, PlanningWorkload, Query, QueryRequirements, - QueryWorkload, SourceFacts, WorkloadError, + QueryWorkload, RangeEvaluation, SourceFacts, WorkloadError, }; use super::*; @@ -366,4 +367,46 @@ mod tests { ) .is_empty()); } + + /// `@ end()` and `@ start()` are lifted above the selector they anchor, + /// which is lowered without them (its offset stays): the selector of + /// `sum_over_time(data[5m] @ start() offset 1m)` is + /// `TimeShift(@ start())` over the selector of + /// `sum_over_time(data[5m] offset 1m)`, so the two share it. Instant and + /// range-evaluated queries lower alike. + #[test] + fn at_start_and_at_end_are_lifted_above_the_selector_they_anchor() { + let lowered = |query: &str| { + let roots = lower_promql_query_workload(&workload(query)).unwrap(); + let [QueryRoot::Operator(root)] = &roots[..] else { + panic!("one operator root"); + }; + Rc::clone(root.children()[0]) + }; + let pinned = lowered("sum_over_time(data[5m] @ start() offset 1m)"); + let NonASAPOp::TimeShift { shift, child } = pinned.expect_non_asap() else { + panic!("expected the lifted TimeShift, got {pinned:?}"); + }; + assert_eq!( + *shift, + TimeShift { + offset_ms: 0, + at: Some(AtModifier::Start), + } + ); + assert_eq!(*child, lowered("sum_over_time(data[5m] offset 1m)")); + + let mut range_query = workload("sum_over_time(data[5m] @ start() offset 1m)"); + if let Some(batch) = range_query.query_workload.query_batch.as_mut() { + batch[0].range_evaluation = Some(RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }); + } + assert_eq!( + lower_promql_query_workload(&range_query).unwrap(), + lower_promql_query_workload(&workload("sum_over_time(data[5m] @ start() offset 1m)")) + .unwrap() + ); + } } diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 8970cdff8..942593d70 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -55,7 +55,8 @@ //! | `m{f}` / `m{f}[w]` | `TimeRange{ingestion, Instant, Scan{predicates}}` / `TimeRange{w, Range, Scan}` | //! | `a OP b` | `BinaryOp{vector_match}` | //! | `expr[r:res]` | `PromqlSubquery{r, res}` | -//! | ` offset ` / ` @ `/`start()`/`end()` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | +//! | ` offset ` / ` @ ` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | +//! | ` @ start()`/`end()` (also a subquery's) | `TimeShift{at: Start \| End}` above the selector `X`, lowered without the `@` (its offset stays in `X`), so `X` is the group of the same selector without `@` | use std::collections::BTreeMap; use std::rc::Rc; @@ -234,14 +235,16 @@ impl PromqlLowerer<'_> { }; // `offset`/`@` move the whole subquery, including its step grid. let shift = time_shift(sq.offset.as_ref(), sq.at.as_ref())?; - Ok(if shift.is_identity() { - subquery - } else { - Unresolved::TimeShift { - shift, - child: Rc::new(subquery), + Ok(anchored(shift, |shift| { + if shift.is_identity() { + subquery + } else { + Unresolved::TimeShift { + shift, + child: Rc::new(subquery), + } } - }) + })) } Expr::VectorSelector(vs) => { let (metric, matchers, shift) = vs_parts(vs)?; @@ -249,11 +252,13 @@ impl PromqlLowerer<'_> { } Expr::MatrixSelector(ms) => { let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok(Unresolved::TimeRange { - range: ms.range, - kind: TimeRangeKind::Range, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) + Ok(selector( + metric, + matchers, + shift, + ms.range, + TimeRangeKind::Range, + )) } // Scalar-typed, dispatched above; kept for exhaustiveness. String // literals only appear as function args (`label_replace`, …), so a @@ -1120,11 +1125,13 @@ impl PromqlLowerer<'_> { intent: AggIntent, ) -> Result { let child = match inner.window { - Some(w) => Unresolved::TimeRange { - range: w, - kind: TimeRangeKind::Range, - child: Rc::new(filtered_source(inner.metric, inner.matchers, inner.shift)), - }, + Some(w) => selector( + inner.metric, + inner.matchers, + inner.shift, + w, + TimeRangeKind::Range, + ), None => self.instant_source(inner.metric, inner.matchers, inner.shift)?, }; let reduction = reduction_for(&keys, inner.window.is_some() || intent.is_per_series()); @@ -1150,11 +1157,52 @@ impl PromqlLowerer<'_> { shift: TimeShift, ) -> Result { let interval = self.sample_interval(&metric)?; - Ok(Unresolved::TimeRange { - range: interval, - kind: TimeRangeKind::Instant, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) + Ok(selector( + metric, + matchers, + shift, + interval, + TimeRangeKind::Instant, + )) + } +} + +/// A selector over `metric`: the `TimeRange` of `range` and `kind` over its +/// filtered `Scan`, shifted by `shift` ([`anchored`]). +fn selector( + metric: String, + matchers: Vec, + shift: TimeShift, + range: Duration, + kind: TimeRangeKind, +) -> Unresolved { + anchored(shift, |shift| Unresolved::TimeRange { + range, + kind, + child: Rc::new(filtered_source(metric, matchers, shift)), + }) +} + +/// The subtree `lower` builds with `shift`, its `@ start()` / `@ end()` +/// lifted above it: `lower` gets `shift` without them (only the offset and +/// an `@ ` stay in the subtree), and `TimeShift { at: Start | End }` +/// reads its result. The subtree is then the group of the same expression +/// without `@`, shared with any query over it; in a range query it runs +/// once per request, at the request's end (or start), and in an instant +/// query both are the evaluation time. +fn anchored(shift: TimeShift, lower: impl FnOnce(TimeShift) -> Unresolved) -> Unresolved { + match shift.at { + Some(at @ (AtModifier::Start | AtModifier::End)) => Unresolved::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(at), + }, + child: Rc::new(lower(TimeShift { + offset_ms: shift.offset_ms, + at: None, + })), + }, + _ => lower(shift), } } diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index 4ea487c36..232495f3a 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -40,7 +40,7 @@ use asap_frontend_promql::PromqlError as LoweringError; use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{ AggIntent, AtModifier, BinaryOpKind, PromQLVectorSetOpKind, Reduction, SampleKind, Source, - TimeFunc, + TimeFunc, TimeShift, }; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::schema::DataType; @@ -1039,7 +1039,9 @@ fn offset_modifier_lowers_to_a_time_shift() { fn at_modifier_lowers_to_a_time_shift() { // SEMANTICS (PromQL, issue #40): `@ ` pins the evaluation to an absolute // instant (PromQL seconds → IR milliseconds); `@ start()` / `@ end()` anchor - // to the query range bounds. + // to the query range bounds: a `TimeShift { at }` above the selector, which + // is lowered without the `@`, so it is the selector of the same query + // without `@`. let qe = ok("http_requests_total @ 1609746000"); let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { panic!("expected an ingestion TimeRange"); @@ -1051,24 +1053,27 @@ fn at_modifier_lowers_to_a_time_shift() { assert_eq!(shift.offset_ms, 0); let qe = ok("http_requests_total @ start()"); - let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { - panic!("expected an ingestion TimeRange"); - }; - let NonASAPOp::TimeShift { shift, .. } = child.expect_non_asap() else { - panic!("expected a TimeShift for `@ start()`"); + let NonASAPOp::TimeShift { shift, child } = qe.expect_non_asap() else { + panic!("expected a TimeShift for `@ start()`, got {qe:?}"); }; - assert_eq!(shift.at, Some(AtModifier::Start)); + assert_eq!( + *shift, + TimeShift { + offset_ms: 0, + at: Some(AtModifier::Start) + } + ); + assert_eq!(**child, *ok("http_requests_total")); - // Offset and `@` compose: `@ end() offset 5m` carries both. + // Offset and `@` compose: `@ end() offset 5m` is `@ end()` over the + // selector with `offset 5m`. let qe = ok("http_requests_total @ end() offset 5m"); - let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { - panic!("expected an ingestion TimeRange, got {qe:?}"); - }; - let NonASAPOp::TimeShift { shift, .. } = child.expect_non_asap() else { + let NonASAPOp::TimeShift { shift, child } = qe.expect_non_asap() else { panic!("expected a TimeShift, got {qe:?}"); }; assert_eq!(shift.at, Some(AtModifier::End)); - assert_eq!(shift.offset_ms, 300_000); + assert_eq!(shift.offset_ms, 0); + assert_eq!(**child, *ok("http_requests_total offset 5m")); } #[test] diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs index 7e706c414..4e51a9dc9 100644 --- a/crates/frontend-sql/src/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -1,5 +1,6 @@ use std::rc::Rc; +use datafusion::arrow::datatypes::{DataType as ArrowDataType, TimeUnit}; use datafusion::logical_expr::utils::{split_binary, split_conjunction}; use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; @@ -137,17 +138,27 @@ impl SqlLowerer<'_> { Expr::IsNotNull(inner) => Ok(Unresolved::IsNotNull(bx(inner)?)), - // Fold a numeric cast of a literal, as DataFusion's constant folding - // does: type coercion turns `latency < 100` on a DOUBLE column into - // `latency < CAST(100 AS DOUBLE)`, which lowers as `latency < 100.0`. + // Fold a numeric or timestamp cast of a literal, as DataFusion's + // constant folding does: type coercion turns `latency < 100` on a + // DOUBLE column into `latency < CAST(100 AS DOUBLE)`, which lowers + // as `latency < 100.0`, and `ts >= '2026-11-01'` into a cast of the + // string, which lowers as a `Timestamp` literal (ms since the epoch). Expr::Cast(c) if matches!(c.expr.as_ref(), Expr::Literal(v, _) if !v.is_null()) - && (c.field.data_type().is_integer() || c.field.data_type().is_floating()) => + && (c.field.data_type().is_integer() + || c.field.data_type().is_floating() + || matches!(c.field.data_type(), ArrowDataType::Timestamp(..))) => { let Expr::Literal(sv, _) = c.expr.as_ref() else { unreachable!() }; - match sv.cast_to(c.field.data_type()) { + let to = match c.field.data_type() { + ArrowDataType::Timestamp(..) => { + ArrowDataType::Timestamp(TimeUnit::Millisecond, None) + } + other => other.clone(), + }; + match sv.cast_to(&to) { Ok(folded) => scalar_value_to_asap(&folded).map(Unresolved::Literal), Err(_) => Ok(Unresolved::Cast { expr: bx(&c.expr)?, diff --git a/crates/frontend-sql/src/sql/types.rs b/crates/frontend-sql/src/sql/types.rs index cf18399ad..08cb21fa9 100644 --- a/crates/frontend-sql/src/sql/types.rs +++ b/crates/frontend-sql/src/sql/types.rs @@ -53,6 +53,13 @@ pub(super) fn scalar_value_to_asap(sv: &DfScalarValue) -> Result Ok(ScalarValue::Utf8(s.clone())), DfScalarValue::Boolean(Some(b)) => Ok(ScalarValue::Boolean(*b)), + // A timestamp is ms since the epoch, as the executor's columns. + DfScalarValue::TimestampSecond(Some(s), _) => Ok(ScalarValue::Timestamp(s * 1_000)), + DfScalarValue::TimestampMillisecond(Some(ms), _) => Ok(ScalarValue::Timestamp(*ms)), + DfScalarValue::TimestampMicrosecond(Some(us), _) => Ok(ScalarValue::Timestamp(us / 1_000)), + DfScalarValue::TimestampNanosecond(Some(ns), _) => { + Ok(ScalarValue::Timestamp(ns / 1_000_000)) + } // All three of DataFusion's interval scalars land on one canonical // shape; the narrower two simply leave the fields they do not carry // at zero. diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 84829c04b..2a83de38e 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -2210,7 +2210,9 @@ async fn lead_in_frame_lowers_to_its_own_kind_not_lead() { /// PromQL's Float64 Unix-seconds `EvalTimestamp`. #[tokio::test] async fn now_in_predicate_lowers_to_current_timestamp() { - // WHERE folds onto Scan.predicates (no explicit Filter node). + // WHERE folds onto Scan.predicates (no explicit Filter node). The + // coercion cast of `now()` to the column's timestamp type is dropped: + // `now()` is already a Timestamp. let qe = lower("SELECT * FROM metrics WHERE ts < NOW()").await; let NonASAPOp::Project { child, .. } = op(&qe) else { panic!("expected Project at root, got {qe:?}"); @@ -2221,7 +2223,7 @@ async fn now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "NOW() must lower to CurrentTimestamp, got {:?}", predicates[0].0 ); @@ -2259,7 +2261,7 @@ async fn clickhouse_now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "now() must lower to CurrentTimestamp, got {:?}", predicates[0].0 ); @@ -3199,3 +3201,35 @@ async fn trailing_window_on_an_aliased_table_and_under_asap_rate() { let (window, _) = trailing_window(below).expect("the WHERE window below it"); assert_eq!(window.as_millis(), 300_000); } + +// An absolute timestamp filter compares the time column with a `Timestamp` +// literal (ms since the epoch), so summary coverage lifts it into an +// interval on the timestamp column: the cast of the string folds, and the +// cast of the Timestamp column to DataFusion's nanosecond unit is dropped. +#[tokio::test] +async fn an_absolute_timestamp_filter_compares_the_column_with_a_timestamp_literal() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + for sql in [ + "SELECT COUNT(*) FROM metrics WHERE ts >= '2026-11-01'", + "SELECT COUNT(*) FROM metrics WHERE ts >= TIMESTAMP '2026-11-01 00:00:00'", + ] { + let qe = lower(sql).await; + let scan = OperatorNode::reachable(&qe) + .into_iter() + .find(|node| matches!(node.non_asap(), Some(NonASAPOp::Scan { .. }))) + .unwrap_or_else(|| panic!("no Scan for {sql}")); + let NonASAPOp::Scan { predicates, .. } = op(&scan) else { + unreachable!() + }; + assert_eq!( + predicates.iter().map(|pred| &pred.0).collect::>(), + [&ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Ge, + right: Box::new(ScalarExpr::Literal(ScalarValue::Timestamp(NOV_1_2026_MS))), + semantics: ExprSemantics::Sql, + }], + "{sql}" + ); + } +} diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index 5227ca379..9ba31b4b1 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -1,30 +1,650 @@ //! `asap-stage0-memo` — #509 Stage 0: the memo groups of the workload's //! query roots (a [`Memo`] without candidates). Each node of the queries' //! DAGs becomes the group of its expression, found or added by -//! [`Memo::find_or_add_group`], so identical sub-expressions are one group. -//! Stage 1 starts from these groups. +//! [`Memo::find_or_add_group`], so identical sub-expressions, within one +//! query or across queries, are one group. Stage 1 starts from these groups. //! -//! It depends only on `asap-types`, where its output (`Memo`) lives; no -//! other stage crate depends on it (the frontends' tests aside, listed in -//! `crate_graph.rs`), and the planner facade calls it first. +//! It depends only on `asap-types`, where its output (`Memo`) lives and +//! where the one expression equality is; no other stage crate depends on +//! it, and the planner facade calls it first. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::rc::Rc; use asap_types::accuracy::AccuracyTarget; -use asap_types::ir::{QueryRoot, SchemaDerivationError}; -use asap_types::stages::Memo; -use asap_types::workload::DataWorkload; +use asap_types::ir::operator::SourceColumn; +use asap_types::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, SchemaDerivationError}; +use asap_types::stages::{Memo, MemoGroupId, MemoQuery, ValueSign}; +use asap_types::workload::bounds::{series_read, source_name, values_per_result}; +use asap_types::workload::{DataWorkload, PrometheusMetricType}; /// Stage 0: the memo groups of the workload's query roots, query `i` /// being `roots[i]` with accuracy target `accuracy_targets[i]` (`None`: /// none), every group without candidates. Each query is added with its -/// $n_q$, the number of values one evaluation returns, which `data` bounds; -/// a `Scan`'s column signs are `data`'s declared ones. Fails with the first -/// node that breaks the node rules over its children's groups -/// ([`Memo::find_or_add_group`]). +/// $n_q$, the number of values one evaluation returns, which `data` bounds +/// ([`values_per_result`]); a `Scan`'s column signs are the ones `data` +/// declares ([`declared_column_signs`]). Fails with the first node that +/// breaks the node rules over its children's groups +/// ([`Memo::find_or_add_group`]); the facade validates the roots first, so +/// that is a frontend bug. pub fn memo_of_queries( roots: &[QueryRoot], accuracy_targets: &[Option], data: &DataWorkload, ) -> Result { - let _ = (roots, accuracy_targets, data); - todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group, each query with its target and n_q") + assert_eq!( + roots.len(), + accuracy_targets.len(), + "one accuracy target per query" + ); + let mut memo = Memo::new(declared_column_signs(roots, data)); + let mut group_of_node = GroupOfNode::new(); + for root in roots { + for node in root.operator_refs() { + group_of(&mut memo, &mut group_of_node, node)?; + } + } + let group = |node: &Rc| group_of_node[&Rc::as_ptr(node)]; + for (root, accuracy_target) in roots.iter().zip(accuracy_targets) { + memo.add_query(MemoQuery { + root: match root { + QueryRoot::Operator(node) => QueryRoot::Operator(group(node)), + QueryRoot::Scalar(expr) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |node| group(node))) + } + }, + accuracy_target: accuracy_target.clone(), + number_of_values_per_query_evaluation: values_per_result( + root, + series_read(root, data), + data, + ), + }); + } + Ok(memo) +} + +/// The sign `data` declares for each column of each source the queries +/// scan ([`SourceFacts::column_signs`](asap_types::workload::SourceFacts::column_signs), +/// a histogram's or summary's for its series, [`DataWorkload::series_facts`]); +/// a declared counter's `value` is non-negative without a declaration. +fn declared_column_signs( + roots: &[QueryRoot], + data: &DataWorkload, +) -> BTreeMap { + let mut signs = BTreeMap::new(); + let mut visited = HashSet::new(); + let mut pending: Vec<&Rc> = + roots.iter().flat_map(QueryRoot::operator_refs).collect(); + while let Some(node) = pending.pop() { + if !visited.insert(Rc::as_ptr(node)) { + continue; + } + pending.extend(node.children()); + let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.operator else { + continue; + }; + let source = source_name(source); + let Some((facts, series_type)) = data.series_facts(source) else { + continue; + }; + let column = |name: &str| SourceColumn { + source: source.to_owned(), + column: name.to_owned(), + }; + for (name, sign) in &facts.column_signs { + signs.insert(column(name), *sign); + } + if series_type == Some(PrometheusMetricType::Counter) { + signs + .entry(column("value")) + .or_insert(ValueSign::NonNegative); + } + } + signs +} + +/// Each node of the queries' DAGs already visited, by address, to its +/// group: a node reached through several paths is visited once. The roots +/// outlive the map, so no address is reused. +type GroupOfNode = HashMap<*const OperatorNode, MemoGroupId>; + +/// The group of `node`: its operator over its children's groups (each +/// found or added first), found or added by [`Memo::find_or_add_group`]. +fn group_of( + memo: &mut Memo, + group_of_node: &mut GroupOfNode, + node: &Rc, +) -> Result { + if let Some(&group) = group_of_node.get(&Rc::as_ptr(node)) { + return Ok(group); + } + for child in node.children() { + group_of(memo, group_of_node, child)?; + } + let expression = node + .operator + .map_children(|child| group_of_node[&Rc::as_ptr(child)]); + let group = memo.find_or_add_group(expression)?; + group_of_node.insert(Rc::as_ptr(node), group); + Ok(group) +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::accuracy::AccuracyTarget; + use asap_types::ir::operator::agg_intent::AggIntent; + use asap_types::ir::operator::arguments::{BinaryOpKind, GroupKeys, Reduction, Source}; + use asap_types::ir::operator::asap::ASAPOp; + use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; + use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; + use asap_types::ir::{BinaryOperator, NonASAPOp, Operator, ScalarExpr}; + use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + + fn node(op: NonASAPOp) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(op)).unwrap() + } + + /// `[ts, service, value, latency]`, no unique key. + fn scan() -> Rc { + node(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) + } + + fn quantile_agg(by: Vec, col: Option, q: f64) -> Rc { + node(NonASAPOp::Aggregate { + reduction: Reduction::by(by), + measures: vec![AggIntent::Quantile { col, q }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }) + } + + fn compare(lhs: Rc, rhs: Rc) -> Rc { + node(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Compare(CompareOpKind::Eq), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) + } + + /// The memo of `roots`, one query per root, without accuracy targets + /// or data facts. + fn memo_of_roots(roots: &[QueryRoot]) -> Result { + memo_of_queries(roots, &vec![None; roots.len()], &DataWorkload::default()) + } + + /// [`memo_of_roots`] of operator roots. + fn memo_of(roots: Vec>) -> Memo { + let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); + memo_of_roots(&roots).unwrap() + } + + /// Query `query`'s root group. + fn root(memo: &Memo, query: usize) -> MemoGroupId { + match &memo.queries[query].root { + QueryRoot::Operator(group) => *group, + QueryRoot::Scalar(_) => panic!("expected an operator root"), + } + } + + /// Whether queries 0 and 1 of `[a, b]` are one group. + fn one_group(a: Rc, b: Rc) -> bool { + let memo = memo_of(vec![a, b]); + root(&memo, 0) == root(&memo, 1) + } + + /// Two queries sharing a sub-expression read one group for it, and + /// every group lists the queries that reach it and the groups that + /// read it. + #[test] + fn two_queries_sharing_a_sub_expression_get_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let p50_vs_p90 = compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.9), + ); + let memo = memo_of(vec![p50, p50_vs_p90]); + // scan, p50, p90, compare. + assert_eq!(memo.groups().len(), 4); + let (scan, p50, compare) = (MemoGroupId(0), root(&memo, 0), root(&memo, 1)); + let p90 = MemoGroupId(2); + assert_eq!( + memo.groups()[compare.0 as usize].expression.children(), + vec![&p50, &p90] + ); + assert_eq!(memo.groups()[scan.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p50.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p90.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[compare.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[scan.0 as usize].parent_groups(), &[p50, p90]); + assert_eq!(memo.groups()[p50.0 as usize].parent_groups(), &[compare]); + assert!(memo + .groups() + .iter() + .all(|group| group.candidates.is_empty())); + } + + /// One query's repeated sub-expression is one group, read twice. + #[test] + fn a_query_s_repeated_sub_expression_is_one_group() { + let memo = memo_of(vec![compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.5), + )]); + assert_eq!(memo.groups().len(), 3); + let p50 = MemoGroupId(1); + assert_eq!( + memo.groups()[root(&memo, 0).0 as usize] + .expression + .children(), + vec![&p50, &p50] + ); + assert_eq!( + memo.groups()[p50.0 as usize].parent_groups(), + &[root(&memo, 0)] + ); + } + + /// A node reached through two paths is one group, visited once. + #[test] + fn an_already_shared_node_is_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let memo = memo_of(vec![compare(Rc::clone(&p50), p50)]); + assert_eq!(memo.groups().len(), 3); + } + + /// Aggregates that differ in a column, a quantile or a filter are + /// different groups. + #[test] + fn aggregates_that_differ_in_an_argument_are_different_groups() { + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::Predicate; + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(3), 0.5) + )); + assert!(!one_group( + quantile_agg(vec![], Some(2), 0.5), + quantile_agg(vec![], Some(2), 0.9) + )); + let filtered = node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![1]), + measures: vec![AggIntent::Quantile { + col: Some(2), + q: 0.5, + }], + output_names: vec![], + filters: vec![Some(Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(3)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: asap_types::ir::ExprSemantics::Sql, + }))], + having: None, + child: scan(), + }); + assert!(!one_group(quantile_agg(vec![1], Some(2), 0.5), filtered)); + } + + /// Equal nodes without a unique key are one group when they return the + /// same rows each time: a keyless scan, a global aggregate, a `without` + /// aggregate. + #[test] + fn equal_keyless_nodes_that_return_the_same_rows_are_one_group() { + let table_scan = || { + node(NonASAPOp::Scan { + source: Source::Table { + table_ref: "metrics".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, true), + ], + 0, + vec![], + ), + }) + }; + assert!(one_group(table_scan(), table_scan())); + assert!(one_group( + quantile_agg(vec![], Some(2), 0.9), + quantile_agg(vec![], Some(2), 0.9) + )); + let without = || { + node(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::without(vec![0])), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }) + }; + assert!(one_group(without(), without())); + } + + /// Queries that differ only in their accuracy target are one group: the + /// target is the query's ([`MemoQuery::accuracy_target`]), not an + /// argument of its operators. + #[test] + fn queries_that_differ_only_in_their_accuracy_target_are_one_group() { + let roots = [ + QueryRoot::Operator(quantile_agg(vec![1], Some(2), 0.99)), + QueryRoot::Operator(quantile_agg(vec![1], Some(2), 0.99)), + ]; + let targets = [ + Some(AccuracyTarget::max_error(0.01)), + Some(AccuracyTarget::max_error(0.05)), + ]; + let memo = memo_of_queries(&roots, &targets, &DataWorkload::default()).unwrap(); + assert_eq!(root(&memo, 0), root(&memo, 1)); + assert_eq!(memo.groups().len(), 2); + assert_eq!(memo.queries[0].accuracy_target, targets[0]); + assert_eq!(memo.queries[1].accuracy_target, targets[1]); + } + + /// `vector(scalar(sum by (service) (m)))`. + fn scalar_of_vector() -> Rc { + let sum_up = node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![1]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }); + node(NonASAPOp::PromqlVectorFromScalar( + ScalarExpr::PromqlScalarFromVector(sum_up), + )) + } + + /// A vector read through `scalar(v)` is a child group like any input, + /// one group across queries, and a scalar query's root reads it. + #[test] + fn a_scalar_referenced_vector_is_one_group_across_queries() { + let roots = vec![ + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(Rc::clone( + scalar_of_vector().children()[0], + ))), + ]; + let memo = memo_of_roots(&roots).unwrap(); + // scan, sum, bridge. + assert_eq!(memo.groups().len(), 3); + assert_eq!(root(&memo, 0), root(&memo, 1)); + let sum = MemoGroupId(1); + assert_eq!(memo.queries[2].root.operator_refs(), vec![&sum]); + assert_eq!( + memo.groups()[sum.0 as usize].reached_by_queries(), + &[0, 1, 2] + ); + } + + /// `SummaryAgg[DDSketch](scan)`. + fn summary_agg() -> Rc { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: scan(), + summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + })) + .unwrap() + } + + /// Two quantile readouts of one summary are two groups over one state + /// group. + #[test] + fn readouts_share_their_state_group_but_not_each_other() { + let readout = |q| { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: summary_agg(), + query: SketchStatistic::Quantile { q }, + })) + .unwrap() + }; + let memo = memo_of(vec![readout(0.95), readout(0.99)]); + let (p95, p99) = (root(&memo, 0), root(&memo, 1)); + assert_ne!(p95, p99); + assert_eq!( + memo.groups()[p95.0 as usize].expression.children(), + memo.groups()[p99.0 as usize].expression.children() + ); + assert_eq!(memo.groups().len(), 4); + } + + /// `DISTINCT` on every column returns the same rows each time and is one + /// group; a `Dedup` on some columns keeps an unspecified row per key. + #[test] + fn dedup_is_one_group_only_on_every_column() { + let dedup = |cols: Vec| { + node(NonASAPOp::Dedup { + cols, + child: scan(), + }) + }; + assert!(one_group(dedup(vec![]), dedup(vec![]))); + assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); + } + + fn unordered_limit(child: Rc) -> Rc { + node(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child, + }) + } + + /// Two unordered `LIMIT 10`s may return different rows: two groups, and + /// so are equal parents over them. + #[test] + fn unordered_limits_and_their_parents_are_different_groups() { + assert!(!one_group(unordered_limit(scan()), unordered_limit(scan()))); + let distinct_over_limit = || { + node(NonASAPOp::Dedup { + cols: vec![], + child: unordered_limit(scan()), + }) + }; + assert!(!one_group(distinct_over_limit(), distinct_over_limit())); + } + + /// A `LIMIT` over a `Sort` with the same partitions keeps the first rows + /// of a fixed order (one group); over a `Sort` partitioned differently, + /// which rows come first depends on input order (two groups). + #[test] + fn limits_over_a_sort_with_their_partitions_are_one_group() { + let limit = |sort_partitions: GroupKeys| { + node(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: node(NonASAPOp::Sort { + keys: vec![asap_types::ir::scalar::SortKey { + expr: ScalarExpr::Column(2), + ascending: false, + nulls_first: false, + }], + partition_by: sort_partitions, + child: scan(), + }), + }) + }; + assert!(one_group( + limit(GroupKeys::default()), + limit(GroupKeys::default()) + )); + assert!(!one_group( + limit(GroupKeys::by(vec![1])), + limit(GroupKeys::by(vec![1])) + )); + } + + /// A projection emitting the literal `value`. + fn literal(value: f64) -> Rc { + node(NonASAPOp::Project { + cols: vec![asap_types::ir::ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::literal_f64(value), + }], + qualifier: None, + child: scan(), + }) + } + + /// Signed zeros and different non-finite literals are different groups; + /// JSON's `null` for non-finite floats never decides equality. + #[test] + fn signed_zero_and_non_finite_literals_are_different_groups() { + for (a, b) in [ + (0.0, -0.0), + (-0.0, 0.0), + (f64::INFINITY, f64::NEG_INFINITY), + (f64::NAN, f64::NAN), + ] { + assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); + } + assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); + } + + /// A frontend node that breaks the node rules (a readout of raw rows + /// instead of summary state) makes Stage 0 return the error. + #[test] + fn memo_of_queries_returns_the_error_of_a_node_that_breaks_the_node_rules() { + let readout_of_rows = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: scan(), + query: SketchStatistic::Quantile { q: 0.5 }, + })) + .unwrap(); + assert!(memo_of_roots(&[QueryRoot::Operator(readout_of_rows)]).is_err()); + } + + /// `m{…}` at the evaluation time: the latest sample of each series. + fn instant_selector(source: Rc) -> Rc { + node(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(60), + kind: asap_types::ir::TimeRangeKind::Instant, + child: source, + }) + } + + /// Each query enters the memo with $n_q$, the values one evaluation + /// returns, as the data workload bounds it: `m`'s 1000 series for the + /// instant vector `m`, one for the scalar `scalar(sum(m))`, unknown for + /// `m` when no series count is declared. + #[test] + fn each_query_enters_with_the_values_one_evaluation_returns() { + let roots = [ + QueryRoot::Operator(instant_selector(scan())), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(Rc::clone( + scalar_of_vector().children()[0], + ))), + ]; + let declared = DataWorkload { + sources: BTreeMap::from([( + "m".to_owned(), + asap_types::workload::SourceFacts { + series_count: Some(1000), + ..Default::default() + }, + )]), + ..Default::default() + }; + let memo = memo_of_queries(&roots, &[None, None], &declared).unwrap(); + assert_eq!( + memo.queries[0].number_of_values_per_query_evaluation, + Some(1000) + ); + assert_eq!( + memo.queries[1].number_of_values_per_query_evaluation, + Some(1) + ); + let undeclared = memo_of_roots(&roots).unwrap(); + assert_eq!( + undeclared.queries[0].number_of_values_per_query_evaluation, + None + ); + } + + /// A `Scan`'s column signs are the data workload's declared ones: `m` + /// declares `latency` non-negative, the counter `requests_total`'s + /// `value` is non-negative without a declaration, and every other + /// column is `Unknown`. + #[test] + fn a_scan_reads_the_declared_column_signs() { + let counter = node(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "requests_total".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }); + let data = DataWorkload { + sources: BTreeMap::from([ + ( + "m".to_owned(), + asap_types::workload::SourceFacts { + column_signs: BTreeMap::from([( + "latency".to_owned(), + ValueSign::NonNegative, + )]), + ..Default::default() + }, + ), + ( + "requests_total".to_owned(), + asap_types::workload::SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + ..Default::default() + }, + ), + ]), + ..Default::default() + }; + let roots = [QueryRoot::Operator(scan()), QueryRoot::Operator(counter)]; + let memo = memo_of_queries(&roots, &[None, None], &data).unwrap(); + use ValueSign::{NonNegative, Unknown}; + let signs = |query| { + &memo.groups()[root(&memo, query).0 as usize] + .group_properties + .column_sign + }; + // [ts, service, value, latency] + assert_eq!(signs(0), &[Unknown, Unknown, Unknown, NonNegative]); + // [ts, value] + assert_eq!(signs(1), &[Unknown, NonNegative]); + } } diff --git a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs index d66060f30..4bf2944bc 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs @@ -42,7 +42,8 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use asap_types::ir::operator::arguments::{Reduction, Source}; +use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::input::OperatorInput; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; use asap_types::ir::schema::FieldDataType; @@ -736,7 +737,7 @@ fn prune_topk_heaps_over_one_row_per_item( } // A PromQL top-k ranks an instant vector: one row per series or // group. (A SQL top-k keeps its heaps.) - if !reads_only_time_series(child) { + if !child.reads_only_time_series() { continue; } prune_own(&mut target.ways_to_compute, |candidate| { @@ -746,14 +747,6 @@ fn prune_topk_heaps_over_one_row_per_item( } } -/// Whether every scan below `node` reads a time series (PromQL rows). -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node.children().iter().all(|c| reads_only_time_series(c)), - } -} - /// The input, update and partitions of a whole-expression top-k over /// `target`'s inner aggregate, by the keyed-additive rule, or `None` when it /// does not apply. Rows that carry the full series identity rank it as a diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index e2a6ef1da..603f4a4aa 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -178,6 +178,16 @@ pub enum Source { Table { table_ref: String }, } +/// A column of a source (a `Scan`'s metric or table), by name: what the data +/// workload declares facts about, such as a column's sign +/// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceColumn { + pub source: String, + pub column: String, +} + /// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's /// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison /// operator has exactly one representation (and one `Display`) across the IR. diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index b5d0ac29d..caf767173 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -6,6 +6,7 @@ use std::rc::Rc; +use super::arguments::Source; use super::node::{Operator, OperatorNode, OperatorResultKind}; use super::non_asap::NonASAPOp; use crate::ir::properties::summary_coverage::SummaryCoverage; @@ -21,6 +22,25 @@ pub trait OperatorInput: Sized + Clone { /// What the input covers when it is an aggregate group in the memo; /// `None` for any other input. fn aggregate_coverage(&self) -> Option<&SummaryCoverage>; + /// Whether every `Scan` below it reads a time series (PromQL rows): + /// [`reads_only_time_series`] of its operator, a memo group's stored + /// property. + fn reads_only_time_series(&self) -> bool; +} + +/// Whether every `Scan` at or below `operator` reads a time series: a +/// `Scan` of a `Source::TimeSeries`, or any other operator all of whose +/// inputs do. +pub(crate) fn reads_only_time_series(operator: &Operator) -> bool { + match operator { + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { + matches!(source, Source::TimeSeries { .. }) + } + operator => operator + .children() + .into_iter() + .all(OperatorInput::reads_only_time_series), + } } /// A front-end node tree. Node trees carry no summary coverage. @@ -40,6 +60,10 @@ impl OperatorInput for Rc { fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { None } + + fn reads_only_time_series(&self) -> bool { + reads_only_time_series(&self.operator) + } } /// An input of a rule: a real input `I`, or a build the rule constructs @@ -115,4 +139,11 @@ impl OperatorInput for InputOrTemporaryBuild { Self::TemporaryBuild(_) => None, } } + + fn reads_only_time_series(&self) -> bool { + match self { + Self::Input(input) => input.reads_only_time_series(), + Self::TemporaryBuild(build) => reads_only_time_series(&build.operator), + } + } } diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index b9b53b2a4..c8e36b8e4 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -28,7 +28,7 @@ pub use agg_intent::{ }; pub use arguments::{ AtModifier, BinaryOpKind, ColState, GroupKeys, GroupSide, InfoMatcher, JoinKind, - PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, Source, TimeShift, - VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, WindowFrameOffset, - WindowFrameUnits, WindowFuncKind, + PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, Source, SourceColumn, + TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, + WindowFrameOffset, WindowFrameUnits, WindowFuncKind, }; diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index e67a5acc0..651626c34 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -3,15 +3,17 @@ //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! -//! Coverage is derived once for each summary-state memo group, from its -//! input groups, never declared, and stored in its `GroupProperties`. -//! Walking down from a `SummaryAgg` through the input groups' expressions, a +//! Coverage is derived once for each aggregate memo group (a summary state +//! or an exact `Aggregate`), from its input groups, never declared, and +//! stored in its `GroupProperties::aggregate_coverage`. Walking down from a +//! `SummaryAgg` or `Aggregate` through the input groups' expressions, a //! predicate conjunct moves into `selection` when it can be lifted to the -//! `SummaryAgg` (through `Filter`, a range `TimeRange`, a `TimeShift` without -//! `@` and direct-column `Project` items) and it is a value set or an -//! interval on one column. Everything else stays in `definition` as a -//! residual, so two states are merged only when they compute the same thing -//! over disjoint rows. Node trees carry no coverage. +//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and +//! direct-column `Project` items) and it is a value set or an interval on +//! one column. The `TimeRange` and `TimeShift`s become one interval on the +//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! `definition` as a residual, so two states are merged only when they +//! compute the same thing over disjoint rows. Node trees carry no coverage. use std::cmp::Ordering; use std::collections::BTreeMap; use std::ops::Bound; @@ -19,7 +21,7 @@ use std::ops::Bound; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::ir::operator::arguments::Reduction; +use crate::ir::operator::arguments::{AtModifier, Reduction}; use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; @@ -234,12 +236,12 @@ mod limit_bound { #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum CoverageError { - /// The node is not a summary state (`SummaryAgg`, `SummaryTumblingWindowChain` or - /// `SummaryMerge`): a caller bug, or bad input when it is a merge's - /// input. - #[error( - "coverage is derived only for SummaryAgg, SummaryTumblingWindowChain and SummaryMerge" - )] + /// The node has no coverage: it is no aggregate (`SummaryAgg`, + /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a + /// `TimeRange`/`TimeShift` over a chain whose windows are not one + /// interval on the timestamp column. A caller bug, or bad input when it + /// is a merge's input. + #[error("coverage is derived only for aggregates and one time interval over a chain")] NotSummary, /// A `SummaryMerge` without inputs: bad input. #[error("summary merge requires at least one input")] @@ -280,10 +282,15 @@ pub enum CoverageError { type GroupInput<'m> = InputOrTemporaryBuild>; impl SummaryCoverage { - /// Coverage of a summary state a memo group computes with `operator` - /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, + /// Coverage of what a memo group computes with `operator` over its + /// input groups: a `SummaryAgg`, an exact `Aggregate`, a + /// `SummaryTumblingWindowChain` (a build over its input, the timestamp + /// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the + /// chain's coverage with the windows' interval on the timestamp column), /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped - /// `SummaryAgg`. A merge reads its inputs' stored coverage and fails + /// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` + /// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge + /// reads its inputs' stored coverage and fails /// unless every input has the same definition and their selections are /// pairwise disjoint; a family whose merge is idempotent /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may @@ -296,14 +303,41 @@ impl SummaryCoverage { op.map_children(|input| InputOrTemporaryBuild::Input(*input)) }; match operator { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = + unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); + of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if operator.output_kind() == OperatorResultKind::State => + { + of_window_selection(operator) + } Operator::NonASAP(_) => match selected_build(operator)? { Some(build) => of_summary_agg(&build), None => Err(CoverageError::NotSummary), }, Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( - "#1207 PR 2: a chain's coverage is a build's over its input, with the timestamp column unconstrained" - ), + Operator::ASAP(op @ ASAPOp::SummaryTumblingWindowChain { .. }) => { + let ASAPOp::SummaryTumblingWindowChain { + child, + summary_type, + input, + reduction, + filter, + .. + } = as_built(op) + else { + unreachable!("matched a chain"); + }; + of_summary_agg(&ASAPOp::SummaryAgg { + child, + summary_type, + input, + reduction, + filter, + }) + } Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { @@ -316,7 +350,7 @@ impl SummaryCoverage { .ok_or(CoverageError::NotSummary) }) .collect::, _>>()?; - let merged = of_merge(&inputs)?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; match group_by { None => Ok(merged), Some(keys) => regrouped(merged, &children[0], keys), @@ -407,8 +441,12 @@ fn selected_build<'m>( })) } -/// The coverage of a merge of states covering `inputs`. -fn of_merge(inputs: &[SummaryCoverage]) -> Result { +/// The coverage of a merge of states covering `inputs`, whose builds read +/// rows with timestamp column `time_column`. +fn of_merge( + inputs: &[SummaryCoverage], + time_column: Option<&ColumnIdentity>, +) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; if inputs .iter() @@ -426,12 +464,16 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result None, } .unwrap_or(MergeInputRule::Disjoint); - let separated: fn(&SelectionBox, &SelectionBox) -> bool = match required { - MergeInputRule::MayOverlap => |_, _| true, - MergeInputRule::DisjointInTime => todo!( - "#1207 PR 2: disjoint intervals on the timestamp column, with IntervalConstraintLimit ends" - ), - MergeInputRule::Disjoint => SelectionBox::disjoint, + let separated = |a: &SelectionBox, b: &SelectionBox| match required { + MergeInputRule::MayOverlap => true, + // Consecutive windows of one series: disjoint on the timestamp column. + MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { + matches!( + (a.columns.get(column), b.columns.get(column)), + (Some(a), Some(b)) if a.disjoint(b) + ) + }), + MergeInputRule::Disjoint => a.disjoint(b), }; // Every pair of boxes of different inputs. if required != MergeInputRule::MayOverlap @@ -542,9 +584,15 @@ fn build_input_schema(state: &I) -> Option { } } +/// The timestamp column of the rows the build of summary state `state` +/// reads ([`build_input_schema`]), as a selection names it. +fn time_column_of_build_input(state: &I) -> Option { + let rows = build_input_schema(state)?; + column_identity(&rows, rows.time_index?) +} + /// The coverage of `build`, a `SummaryAgg` over memo groups (or over an -/// input a rule built over them): its operators walked down through the -/// liftable operators of its input groups' expressions. +/// input a rule built over them). fn of_summary_agg(build: &ASAPOp>) -> Result { let ASAPOp::SummaryAgg { child, @@ -556,6 +604,26 @@ fn of_summary_agg(build: &ASAPOp>) -> Result, + filter: Option<&Predicate>>, + unlifted_top: impl FnOnce(Option>>) -> Operator>, +) -> Result { // The operators a predicate can be lifted through, top to bottom, and // the first input below them. A subquery's group stays named in the // residual of the operator that reads it. @@ -566,7 +634,11 @@ fn of_summary_agg(build: &ASAPOp>) -> Result true, Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), + // `@ end()` / `@ start()` pin their subtree to one end of the + // request: not lifted, so their group is the input group. + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + !matches!(shift.at, Some(AtModifier::Start | AtModifier::End)) + } _ => false, }; if !passes { @@ -583,9 +655,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>>> = chain .iter() .enumerate() @@ -619,13 +689,26 @@ fn of_summary_agg(build: &ASAPOp>) -> Result None, }; - let mut unlifted_operators = vec![Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - filter: agg_filter, - })]; + // The `TimeRange` and `TimeShift`s: one interval on the timestamp column + // of the `TimeRange`'s input, when they make one and it combines with + // what is lifted on that column already; else they stay unlifted. + let time = time_constraint(&chain).and_then(|constraint| { + let depth = chain + .iter() + .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; + let column = chain[depth].children()[0].schema().time_index?; + let column = column_at_top(column, &chain[..=depth])?; + column_identity(top, column)?; + let combined = match lifted.get(&column) { + Some(existing) => existing.intersect(&constraint)?, + None => constraint, + }; + Some((column, combined)) + }); + let lift_time = time.is_some(); + lifted.extend(time); + + let mut unlifted_operators = vec![unlifted_top(agg_filter)]; for (depth, link) in chain.iter().enumerate() { match link { Operator::NonASAP(NonASAPOp::Filter { .. }) => { @@ -636,9 +719,8 @@ fn of_summary_agg(build: &ASAPOp>) -> Result todo!( - "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with IntervalConstraintLimit ends" - ), + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if lift_time => {} other => unlifted_operators.push(unlifted(other)), } } @@ -653,14 +735,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>) -> Result>, top: &Schema, @@ -688,15 +762,7 @@ fn residual( for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; - let field = &top.fields[column]; - let namesakes = top - .fields - .iter() - .filter(|f| f.name == field.name && f.table == field.table) - .count(); - if namesakes != 1 { - return None; - } + column_identity(top, column)?; let combined = match lifted.get(&column) { Some(existing) => existing.intersect(&constraint)?, None => constraint, @@ -725,8 +791,8 @@ fn group_id(input: &GroupInput<'_>) -> Option { } } -/// `link`, a liftable operator kept whole in a definition: its main input -/// `None`, every other input the group it is. +/// `link`, a liftable operator or an exact `Aggregate` kept whole in a +/// definition: its main input `None`, every other input the group it is. fn unlifted(link: &Operator>) -> Operator> { let mut operator = link.map_children(group_id); match &mut operator { @@ -734,9 +800,10 @@ fn unlifted(link: &Operator>) -> Operator> { NonASAPOp::Filter { child, .. } | NonASAPOp::Project { child, .. } | NonASAPOp::TimeRange { child, .. } - | NonASAPOp::TimeShift { child, .. }, + | NonASAPOp::TimeShift { child, .. } + | NonASAPOp::Aggregate { child, .. }, ) => *child = None, - _ => unreachable!("only liftable operators are walked"), + _ => unreachable!("only liftable operators and aggregates are unlifted"), } operator } @@ -754,6 +821,118 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option Option { + let field = schema.fields.get(column)?; + let namesakes = schema + .fields + .iter() + .filter(|f| f.name == field.name && f.table == field.table) + .count(); + (namesakes == 1).then(|| ColumnIdentity { + table: field.table.clone(), + name: field.name.clone(), + }) +} + +/// The interval on the timestamp column that `links` (top down, the +/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: +/// `(anchor − o − W, anchor − o]` for the one range `TimeRange` of width W +/// and the shifts' offsets summing to o, left-open and right-closed like a +/// PromQL range. The anchor is the evaluation time t +/// (`EvaluationTimePlusMs`), or T for `@ T` (`FixedValue(Timestamp)`). +/// `None` when they are not one interval: no or several range `TimeRange`s, +/// an instant one, several `@`s, or an `@ end()` or `@ start()`, which pins +/// its subtree to one end of the request and is never lifted (the front +/// ends put it above the subtree it anchors). +fn time_constraint(links: &[Operator]) -> Option { + let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); + for link in links { + match link { + Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { + if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { + return None; + } + } + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + offset_ms = offset_ms.checked_add(shift.offset_ms)?; + if let Some(anchor) = shift.at { + if at.replace(anchor).is_some() { + return None; + } + } + } + _ => {} + } + } + let width_ms = i64::try_from(width?.as_millis()).ok()?; + let fixed_anchor = match at { + Some(AtModifier::Timestamp(ms)) => Some(ms), + Some(AtModifier::Start | AtModifier::End) => return None, + None => None, + }; + let before_anchor = |ms: i64| -> Option { + Some(match fixed_anchor { + Some(anchor) => { + IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) + } + None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), + }) + }; + Some(Constraint::Interval { + lower: Bound::Excluded(before_anchor(offset_ms.checked_add(width_ms)?)?), + upper: Bound::Included(before_anchor(offset_ms)?), + }) +} + +/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain +/// (through more of them): the chain's coverage with the interval they +/// select on the timestamp column of the chain's input, in every box. +fn of_window_selection( + selection: &Operator>, +) -> Result { + let mut links = vec![selection.clone()]; + let mut below = *selection.children()[0]; + loop { + match below.operator() { + link @ Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => { + below = child; + links.push(link); + } + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => break, + _ => return Err(CoverageError::NotSummary), + } + } + let chain = below + .aggregate_coverage() + .ok_or(CoverageError::NotSummary)?; + let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NotSummary)?; + let constraint = time_constraint(&links).ok_or(CoverageError::NotSummary)?; + let selection = chain + .selection + .iter() + .map(|selection_box| { + let mut columns = selection_box.columns.clone(); + let combined = match columns.get(&time_column) { + Some(existing) => existing.intersect(&constraint), + None => Some(constraint.clone()), + }; + columns.insert( + time_column.clone(), + combined.ok_or(CoverageError::NotSummary)?, + ); + Ok(SelectionBox { columns }) + }) + .collect::>()?; + Ok(SummaryCoverage { + definition: chain.definition.clone(), + selection, + }) +} + /// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { diff --git a/crates/types/src/ir/schema/series_identity.rs b/crates/types/src/ir/schema/series_identity.rs index 94e997390..61bc53e69 100644 --- a/crates/types/src/ir/schema/series_identity.rs +++ b/crates/types/src/ir/schema/series_identity.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (Stage 0 hash-consing, `cse` in `asap_stage0_memo`). + // node is legal to share (hash-consing, `Memo::find_or_add_group`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 13ffd59b2..065a372fb 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,6 +1,6 @@ //! Ranked-row output contract shared by logical TopK and its implementations. use crate::ir::operator::input::OperatorInput; -use crate::ir::operator::{Reduction, Source}; +use crate::ir::operator::Reduction; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; use crate::ir::schema::{ColumnId, DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; @@ -85,7 +85,7 @@ pub(crate) fn ranked_row_projection( alias: Some(output_name.into()), expr: ScalarExpr::Column(value), }); - let promql = reads_only_time_series(child); + let promql = child.reads_only_time_series(); for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item @@ -142,12 +142,3 @@ pub fn series_identity_item( SummaryInputExpr::Tuple(items) } } - -fn reads_only_time_series(node: &I) -> bool { - match node.operator() { - Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { - matches!(source, Source::TimeSeries { .. }) - } - operator => operator.children().into_iter().all(reads_only_time_series), - } -} diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs index 618c228bc..f1f89ab30 100644 --- a/crates/types/src/stages/logical/candidate.rs +++ b/crates/types/src/stages/logical/candidate.rs @@ -84,7 +84,7 @@ pub enum CandidateError { pub struct LogicalProperties { /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for /// an exact operator. - parameters: Option, + pub(super) parameters: Option, } impl LogicalProperties { diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs new file mode 100644 index 000000000..b2fe79c57 --- /dev/null +++ b/crates/types/src/stages/logical/column_properties.rs @@ -0,0 +1,449 @@ +//! Value sign: the sign of each output column of a memo group (concepts +//! doc, section 3, "Value sign"), derived once from the input groups' signs +//! and, for a `Scan`, the signs the data workload declares for its source's +//! columns. Error contracts that hold only for values of one sign read it. + +use std::collections::BTreeMap; + +use super::{MemoGroupAsInput, ValueSign}; +use crate::ir::operator::arguments::{ + BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction, SourceColumn, +}; +use crate::ir::operator::asap::{summary_agg_schema, summary_build}; +use crate::ir::operator::input::OperatorInput; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; +use crate::ir::schema::{FieldDataType, Schema}; +use crate::ir::{ASAPOp, NonASAPOp, Operator}; +use crate::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; +use crate::workload::bounds::source_name; + +use ValueSign::{NonNegative, NonPositive, Unknown}; + +type Input<'m> = MemoGroupAsInput<'m>; + +/// The sign of each column of `output`, the output schema of `operator`, +/// from its input groups' `column_sign` and, for a `Scan`, `declared`. +pub(crate) fn output_column_sign( + operator: &Operator>, + output: &Schema, + declared: &BTreeMap, +) -> Vec { + let signs = match operator { + Operator::NonASAP(op) => non_asap_signs(op, output, declared), + Operator::ASAP(op) => asap_signs(op, output), + }; + debug_assert_eq!( + signs.len(), + output.fields.len(), + "one sign per output column of {}", + operator.kind_name() + ); + signs +} + +fn signs_of(input: &Input<'_>) -> Vec { + input.group_properties().column_sign.clone() +} + +fn non_asap_signs( + op: &NonASAPOp>, + output: &Schema, + declared: &BTreeMap, +) -> Vec { + use NonASAPOp::*; + let unknown = || vec![Unknown; output.fields.len()]; + match op { + Scan { source, schema, .. } => schema + .fields + .iter() + .map(|field| { + let column = SourceColumn { + source: source_name(source).to_owned(), + column: field.name.clone(), + }; + declared.get(&column).copied().unwrap_or(Unknown) + }) + .collect(), + Values { .. } | PromqlVectorFromScalar(_) => unknown(), + Filter { child, .. } + | Sort { child, .. } + | Limit { child, .. } + | Dedup { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | PromqlSubquery { child, .. } + | PromqlSeriesSample { child, .. } + | PromqlInfoEnrich { child, .. } => signs_of(child), + // The relabeled column is written; every other column is copied. + PromqlRelabel { dst, child, .. } => { + let mut signs = signs_of(child); + match child.schema().column_id(dst) { + Some(written) => signs[written] = Unknown, + None => signs.push(Unknown), + } + signs + } + Project { cols, child, .. } => project_signs(cols, &signs_of(child)), + Aggregate { + reduction, + measures, + child, + .. + } => { + if let [AggIntent::TopK { .. }] = measures.as_slice() { + // The ranked rows, projected: each column copied. + return match crate::ir::schema::topk::ranked_row_projection(child, reduction, "") { + Ok(cols) => project_signs(&cols, &signs_of(child)), + Err(_) => unknown(), + }; + } + aggregate_signs(child, reduction, measures, output) + } + Join { + kind, left, right, .. + } => { + let mut signs = signs_of(left); + if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { + signs.extend(signs_of(right)); + } + signs + } + SetOp { left, right, .. } => agreed(&[signs_of(left), signs_of(right)]), + Concat { children } => agreed(&children.iter().map(signs_of).collect::>()), + SQLWindowFunc { child, .. } => { + let mut signs = signs_of(child); + signs.push(Unknown); + signs + } + // The left side's columns, its value per the operator, then the + // labels the match adds (unknown). + BinaryOp { + operator, lhs, rhs, .. + } => { + let mut signs = signs_of(lhs); + if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { + let rhs_value = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) + .ok() + .map_or(Unknown, |column| signs_of(rhs)[column]); + signs[value] = match &operator.kind { + // A filter keeps the left side's values. + BinaryOpKind::Compare(_) + | BinaryOpKind::Set( + PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless, + ) => signs[value], + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(signs[value], rhs_value), + BinaryOpKind::CompareBool(_) => NonNegative, + BinaryOpKind::Arithmetic(_) => Unknown, + }; + } + signs.resize(output.fields.len(), Unknown); + signs + } + } +} + +fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { + use ASAPOp::*; + match op { + SummaryAgg { + child, + input, + reduction, + .. + } => state_signs(child, input, reduction, output), + // The build's layout without its own time column, after the + // computed `window_end` (column 0). + SummaryTumblingWindowChain { + child, + summary_type, + input, + reduction, + .. + } => match summary_agg_schema(child.schema(), summary_type, reduction) { + Ok(build) => { + let mut signs = state_signs(child, input, reduction, &build); + if let Some(time) = build.time_index { + signs.remove(time); + } + signs.insert(0, Unknown); + signs + } + Err(_) => vec![Unknown; output.fields.len()], + }, + SummaryMerge { children, group_by } => { + // The windows a `TimeRange` over a chain selects merge into the + // chain's build over that range. + if let (None, [child]) = (group_by, children.as_slice()) { + if let Some(SummaryTumblingWindowChain { + child, + input, + reduction, + .. + }) = summary_build(child) + { + return state_signs(&child, &input, &reduction, output); + } + } + let merged = agreed(&children.iter().map(signs_of).collect::>()); + match group_by { + None => merged, + Some(keys) => { + let state = children[0] + .schema() + .fields + .iter() + .position(|field| !field.is_plain()); + keys.iter() + .chain(state.as_ref()) + .map(|&column| merged.get(column).copied().unwrap_or(Unknown)) + .collect() + } + } + } + SummaryEstimate { + summary_input, + query, + } => readout_signs(summary_input, output, |state| match query { + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy => NonNegative, + SketchStatistic::Quantile { .. } + | SketchStatistic::PointCount { .. } + | SketchStatistic::TopK { .. } => state, + }), + FinalizeExactAccumulator { child } => { + let counts = child.schema().fields.iter().any(|field| { + matches!( + field.dtype, + FieldDataType::ExactAggregate(ExactKind::Count, _) + ) + }); + readout_signs( + child, + output, + |state| if counts { NonNegative } else { state }, + ) + } + MaintainPopulation { child, .. } => signs_of(child), + EvaluatePopulation { child, .. } if output == child.schema() => signs_of(child), + EvaluatePopulation { .. } + | SummarySubtract { .. } + | SummaryDelete { .. } + | SummaryJoin { .. } + | Extension { .. } => vec![Unknown; output.fields.len()], + } +} + +/// A column item keeps its input column's sign; an expression item's sign +/// is [`expression_sign`]'s. +fn project_signs(cols: &[ProjectItem], input: &[ValueSign]) -> Vec { + cols.iter() + .map(|item| expression_sign(&item.expr, input)) + .collect() +} + +/// `abs` is non-negative, a negation flips its operand's sign, a numeric +/// cast keeps it and a numeric literal has its own; any other expression is +/// `Unknown`. +fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { + match expr { + ScalarExpr::Column(column) => input.get(*column).copied().unwrap_or(Unknown), + ScalarExpr::Literal(ScalarValue::Float64(value)) => sign_of_number(*value), + ScalarExpr::Literal(ScalarValue::Int64(value)) => sign_of_number(*value as f64), + ScalarExpr::Negative { expr, .. } => flipped(expression_sign(expr, input)), + ScalarExpr::Cast { expr, .. } => expression_sign(expr, input), + // PromQL's `abs(v)` lowers to `promql_abs`, SQL's to `abs`. + ScalarExpr::FunctionCall { name, .. } + if name.eq_ignore_ascii_case("abs") || name == "promql_abs" => + { + NonNegative + } + _ => Unknown, + } +} + +/// An aggregate's, in `aggregate_output_schema`'s layout: a per-entity +/// reduction keeps its input's columns in place, its value replaced by the +/// measure's; a `by` reduction lists its keys, then its measures; a +/// `without` reduction the input's other label columns, then its measures. +fn aggregate_signs( + child: &Input<'_>, + reduction: &Reduction, + measures: &[AggIntent], + output: &Schema, +) -> Vec { + let input = signs_of(child); + let schema = child.schema(); + let sample_value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); + let value_of = |measure: &AggIntent| measure.input_cols().first().copied().or(sample_value); + let keys: Vec = match reduction { + Reduction::PerEntity => { + let mut signs = input.clone(); + if let Some(measure) = measures.first() { + if let Some(value) = value_of(measure).filter(|&value| value < signs.len()) { + signs[value] = measure_sign(measure, input[value]); + } + } + return signs; + } + Reduction::Reduce(by) if by.is_without() => (0..schema.fields.len()) + .filter(|&column| { + schema.time_index != Some(column) + && sample_value != Some(column) + && !by.keys().contains(&column) + }) + .map(|column| input[column]) + .collect(), + Reduction::Reduce(by) => by + .keys() + .iter() + .map(|&key| input.get(key).copied().unwrap_or(Unknown)) + .collect(), + }; + let measured: Vec = measures + .iter() + .flat_map(|measure| { + let sign = measure_sign( + measure, + value_of(measure) + .and_then(|value| input.get(value).copied()) + .unwrap_or(Unknown), + ); + // `count_values` and `absent` add label columns before their value. + let labels = match measure { + AggIntent::CountValues { label } => { + usize::from(!output.fields[..keys.len()].iter().any(|f| f.name == *label)) + } + AggIntent::Absent { labels } => labels.len(), + _ => 0, + }; + std::iter::repeat_n(Unknown, labels).chain([sign]) + }) + .collect(); + keys.into_iter().chain(measured).collect() +} + +/// The sign of `measure`'s value over an input value of sign `input`. +fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { + use AggIntent::*; + match measure { + Count + | Cardinality { .. } + | CountValues { .. } + | Changes + | Resets + | Group + | Absent { .. } + | AbsentOverTime + | PresentOverTime + | StdDev { .. } + | Variance { .. } + | FrequencyL2 { .. } + | FrequencyEntropy { .. } + | Math(crate::ir::operator::MathFunc::Abs) => NonNegative, + Sum { .. } + | Avg { .. } + | Min { .. } + | Max { .. } + | Quantile { .. } + | LastOverTime + | FirstOverTime => input, + // A counter's rate and increase are non-negative; over a value of + // another sign a decrease reads as a reset, so no sign is known. + Rate | IRate | Increase if input == NonNegative => NonNegative, + _ => Unknown, + } +} + +/// A summary state's, in `state_schema`'s layout (an aggregate of one +/// measure): its key columns keep their input's sign and its state column +/// records the sign of the values it is updated with. +fn state_signs( + child: &Input<'_>, + update: &SummaryUpdate, + reduction: &Reduction, + state_schema: &Schema, +) -> Vec { + let mut signs = aggregate_signs( + child, + reduction, + &[AggIntent::Sum { col: None }], + state_schema, + ); + let updated_with = match &update.weight { + SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) + .ok() + .and_then(|column| signs_of(child).get(column).copied()) + .unwrap_or(Unknown), + SummaryInputExpr::Constant(value) => sign_of_number(*value), + SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, + }; + if let Some(state) = state_schema.fields.iter().position(|f| !f.is_plain()) { + signs[state] = updated_with; + } + signs +} + +/// A readout of summary state: the state's plain columns keep their sign +/// in place, and the value read out has `value(the state's recorded sign)`. +/// `Unknown` everywhere when the output does not keep the state's layout (a +/// top-k's rows). +fn readout_signs( + state: &Input<'_>, + output: &Schema, + value: impl Fn(ValueSign) -> ValueSign, +) -> Vec { + let input = state.schema(); + if input.fields.len() != output.fields.len() { + return vec![Unknown; output.fields.len()]; + } + signs_of(state) + .into_iter() + .zip(&input.fields) + .map(|(sign, field)| if field.is_plain() { sign } else { value(sign) }) + .collect() +} + +/// Per column, the inputs' sign if they all agree, else `Unknown`. +fn agreed(inputs: &[Vec]) -> Vec { + let Some((first, rest)) = inputs.split_first() else { + return Vec::new(); + }; + first + .iter() + .enumerate() + .map(|(column, &sign)| { + rest.iter().fold(sign, |agreed, other| { + agree(agreed, other.get(column).copied().unwrap_or(Unknown)) + }) + }) + .collect() +} + +fn agree(left: ValueSign, right: ValueSign) -> ValueSign { + if left == right { + left + } else { + Unknown + } +} + +fn flipped(sign: ValueSign) -> ValueSign { + match sign { + NonNegative => NonPositive, + NonPositive => NonNegative, + Unknown => Unknown, + } +} + +fn sign_of_number(value: f64) -> ValueSign { + if value >= 0.0 { + NonNegative + } else if value < 0.0 { + NonPositive + } else { + Unknown + } +} diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index d00f3f0a6..30fcdab52 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -1,11 +1,14 @@ //! Group scope: memo groups, the distinct logical sub-expressions of the //! workload, and the properties every candidate of a group shares. -use crate::ir::operator::input::OperatorInput; +use super::column_properties::output_column_sign; +use crate::ir::operator::arguments::{AtModifier, TimeShift}; +use crate::ir::operator::input::{reads_only_time_series, OperatorInput}; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::schema::Schema; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; +use crate::ir::{NonASAPOp, ScalarExpr}; use crate::ir::{Operator, OperatorResultKind}; use crate::stages::{CandidateForMemoGroup, Memo}; @@ -25,10 +28,14 @@ pub struct MemoGroup { /// changes. pub group_properties: GroupProperties, /// The groups that read it ([`Self::parent_groups`]) through their - /// expression or a candidate's. + /// expression or a candidate's: kept up to date by + /// [`Memo::find_or_add_group`](super::Memo::find_or_add_group) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). pub parents: Vec, /// The queries whose roots reach it ([`Self::reached_by_queries`]), - /// ascending. + /// ascending: kept up to date by + /// [`Memo::add_query`](super::Memo::add_query) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). pub queries: Vec, /// Its candidates: empty after Stage 0, filled by Stage 1. pub candidates: Vec, @@ -53,22 +60,27 @@ impl MemoGroup { pub struct GroupProperties { pub schema: Schema, pub result_kind: OperatorResultKind, - /// Per output column, in schema order: the sign of its values. + /// Per output column, in schema order: the sign of its values, from the + /// input groups' and, for a `Scan`, the declared ones. pub column_sign: Vec, /// Whether the result changes with the evaluation time: true for a /// group under a `TimeRange` relative to the evaluation time - /// (`sum_over_time(m[5m])`), false for one under a fixed `@ T` or - /// reading no time. A result that does not depend on it is the same at - /// every evaluation, so a stored copy delivers on every grid. + /// (`sum_over_time(m[5m])`) or reading it (`time()`, `now()`), false + /// for one under a fixed `@ T` or reading no time. A result that does + /// not depend on it is the same at every evaluation, so a stored copy + /// delivers on every grid. pub depends_on_evaluation_time: bool, - /// What an aggregate group covers: a summary state, a chain, a merge - /// over a chain's windows, or an exact `Aggregate` result; `None` for - /// any other group. Derived today for summary states only; exact - /// `Aggregate` groups follow in #1207 PR 2. + /// What an aggregate group covers: a summary state, a chain, a + /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact + /// `Aggregate` result; `None` for any other group. pub aggregate_coverage: Option, /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). pub shareable_when_equal: bool, + /// Whether every `Scan` below reads a time series (PromQL rows): a + /// `Scan` of a `Source::TimeSeries`, or any other operator all of whose + /// input groups do. A PromQL top-k ranks one row per series. + pub reads_only_time_series: bool, } /// The sign of one column's values, a group property derived from the @@ -88,9 +100,6 @@ pub enum ValueSign { /// input group's `group_properties` and `expression` /// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression /// breaks them (schema derivation, input validation, merge check). -// `column_sign` and `depends_on_evaluation_time` are derived in #1207 PR 2; -// until then the result is unreachable. -#[allow(unreachable_code)] pub(super) fn derive_group_properties( memo: &Memo, expression: &Operator, @@ -107,15 +116,55 @@ pub(super) fn derive_group_properties( Err(_) => None, }; Ok(GroupProperties { + column_sign: output_column_sign(&operator, &schema, &memo.declared_column_signs), + depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), - column_sign: todo!("#1207 PR 2: each output column's sign, per operator, from the input groups' and the sources' declared signs"), - depends_on_evaluation_time: todo!("#1207 PR 2: whether the result changes with the evaluation time, from the operator and the input groups"), + reads_only_time_series: reads_only_time_series(&operator), }) } +/// Whether `operator`'s result changes with the evaluation time: a +/// `TimeRange` reads the window before it, and `time()` or `now()` read it +/// directly; `@ T` pins its subtree to T. Any other operator depends on it +/// when an input group does. +fn depends_on_evaluation_time(operator: &Operator>) -> bool { + fn reads_evaluation_time(expr: &ScalarExpr) -> bool { + matches!( + expr, + ScalarExpr::EvalTimestamp | ScalarExpr::CurrentTimestamp + ) || expr.children().into_iter().any(reads_evaluation_time) + } + match operator { + Operator::NonASAP(NonASAPOp::TimeRange { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: + TimeShift { + at: Some(AtModifier::Timestamp(_)), + .. + }, + .. + }) => false, + Operator::NonASAP(op) => { + op.scalar_exprs().into_iter().any(reads_evaluation_time) + || operator.children().into_iter().any(depends_on_it) + } + Operator::ASAP( + ASAPOp::SummaryAgg { filter, .. } | ASAPOp::SummaryTumblingWindowChain { filter, .. }, + ) => { + filter.iter().any(|pred| reads_evaluation_time(&pred.0)) + || operator.children().into_iter().any(depends_on_it) + } + Operator::ASAP(_) => operator.children().into_iter().any(depends_on_it), + } +} + +fn depends_on_it(input: &MemoGroupAsInput<'_>) -> bool { + input.group_properties().depends_on_evaluation_time +} + /// A memo group read as an operator's input: its `group_properties` and /// its `expression`, whose inputs are read the same way. #[derive(Clone, Copy)] @@ -128,6 +177,10 @@ impl<'m> MemoGroupAsInput<'m> { fn group(&self) -> &'m MemoGroup { &self.memo.groups()[self.group_id.0 as usize] } + + pub(crate) fn group_properties(&self) -> &'m GroupProperties { + &self.group().group_properties + } } impl OperatorInput for MemoGroupAsInput<'_> { @@ -149,4 +202,8 @@ impl OperatorInput for MemoGroupAsInput<'_> { fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { self.group().group_properties.aggregate_coverage.as_ref() } + + fn reads_only_time_series(&self) -> bool { + self.group().group_properties.reads_only_time_series + } } diff --git a/crates/types/src/stages/logical/memo/hash_consing.rs b/crates/types/src/stages/logical/memo/hash_consing.rs new file mode 100644 index 000000000..3a9925050 --- /dev/null +++ b/crates/types/src/stages/logical/memo/hash_consing.rs @@ -0,0 +1,74 @@ +//! The one equality that makes two expressions one memo group, and two +//! candidates of a group one candidate: [`same_expression`] (operator, +//! arguments and input groups). [`Memo::find_or_add_group`] looks a group up +//! by it, Stage 0 included, and [`Memo::add_candidate`] finds a duplicate by +//! it. Parameters θ are not an argument of any operator (they are a +//! candidate's `LogicalProperties`), so builds of one family over one input +//! that differ only in θ are one group. +//! +//! [`Memo::find_or_add_group`]: super::Memo::find_or_add_group +//! [`Memo::add_candidate`]: super::Memo::add_candidate + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use super::MemoGroupId; +use crate::ir::Operator; +use crate::stages::CandidateForMemoGroup; + +/// `expression` with every input group replaced by `()`: the operator and +/// its arguments. +fn operator_and_arguments(expression: &Operator) -> Operator<()> { + expression.map_children(|_| ()) +} + +/// Whether `left` and `right` are one expression: equal operators and +/// arguments ([`same_value`]) over the same input groups at the same +/// positions. +pub(super) fn same_expression(left: &Operator, right: &Operator) -> bool { + left.children() == right.children() + && same_value( + &operator_and_arguments(left), + &operator_and_arguments(right), + ) +} + +/// A hash consistent with [`same_expression`], used only to narrow the +/// groups compared: SipHash over the canonical JSON of the operator and its +/// arguments, then the input groups. A non-finite `f64` serializes as +/// `null`, which only widens one bucket; [`same_expression`] decides. +pub(super) fn expression_hash(expression: &Operator) -> u64 { + let mut hasher = DefaultHasher::new(); + serde_json::to_string(&operator_and_arguments(expression)) + .unwrap_or_default() + .hash(&mut hasher); + expression.children().hash(&mut hasher); + hasher.finish() +} + +/// Typed equality, confirmed by equal serializations: numeric `PartialEq` +/// alone conflates signed zeros, and `SketchParams`' equality omits the +/// recorded error and failure probability. The serialized check is +/// additional evidence, never a replacement for typed equality (JSON maps +/// non-finite floats to `null`). +pub(super) fn same_value(left: &T, right: &T) -> bool { + left == right + && match (serde_json::to_string(left), serde_json::to_string(right)) { + (Ok(left), Ok(right)) => left == right, + _ => false, + } +} + +/// Whether `left` and `right` are one candidate of a group: the same +/// expression ([`same_expression`]) and logical properties (parameters by +/// [`same_value`]). +pub(super) fn same_expression_and_logical_properties( + left: &CandidateForMemoGroup, + right: &CandidateForMemoGroup, +) -> bool { + same_expression(&left.expression.operator, &right.expression.operator) + && same_value( + &left.logical_properties.parameters, + &right.logical_properties.parameters, + ) +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index cdd7e212d..e9c1c56e0 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -2,17 +2,21 @@ //! over its groups; the operations over the whole memo //! (concepts doc `memo-groups-properties-requirements.md`, section 3; #1203). -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +mod hash_consing; use super::group::derive_group_properties; use crate::accuracy::AccuracyTarget; +use crate::ir::operator::arguments::SourceColumn; use crate::ir::Operator; use crate::ir::QueryRoot; use crate::ir::SchemaDerivationError; use crate::stages::{ CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalRequirements, MemoGroup, - MemoGroupId, + MemoGroupId, ValueSign, }; +use hash_consing::{expression_hash, same_expression, same_expression_and_logical_properties}; /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with /// its candidates ([`MemoGroup::candidates`]: empty after Stage 0), and each @@ -24,6 +28,13 @@ pub struct Memo { groups: Vec, /// The queries, by index in the workload. pub queries: Vec, + /// The groups [`Self::find_or_add_group`] may return, by + /// [`expression_hash`] of their expression: every group whose sharing + /// is legal ([`GroupProperties::shareable_when_equal`]). + groups_by_expression_hash: HashMap>, + /// The sign the data workload declares for each source column it + /// declares one for: a `Scan`'s [`GroupProperties::column_sign`]. + pub(super) declared_column_signs: BTreeMap, } /// One query of the memo: its root over groups, its accuracy target and @@ -41,6 +52,15 @@ pub struct MemoQuery { } impl Memo { + /// A memo without groups whose `Scan`s read `declared_column_signs` + /// (an undeclared source column's sign is `Unknown`). + pub fn new(declared_column_signs: BTreeMap) -> Self { + Self { + declared_column_signs, + ..Self::default() + } + } + /// The groups, by id. pub fn groups(&self) -> &[MemoGroup] { &self.groups @@ -50,34 +70,149 @@ impl Memo { /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 /// adds each query once. pub fn add_query(&mut self, query: MemoQuery) -> usize { - let _ = query; - todo!("#1207 PR 2") + let index = self.queries.len(); + let groups = query.root.operator_refs().into_iter().copied().collect(); + self.queries.push(query); + self.add_queries_below(groups, &[index]); + index } - /// Hash-consing: the group with this expression, added if new, with the - /// `GroupProperties` [`derive_group_properties`] derives. Fails when the - /// expression breaks the node rules (schema derivation, input - /// validation, merge check). + /// Hash-consing: the group whose expression is `expression` + /// (`same_expression`: operator, arguments and input groups), else a + /// new group for it, with the `GroupProperties` [`derive_group_properties`] + /// derives from its input groups, listed as a parent of each input + /// group. A group whose copies may return different rows (an unordered + /// `LIMIT`, [`GroupProperties::shareable_when_equal`]) is never returned + /// for another expression, so such an expression always gets a new + /// group. Fails, leaving the memo unchanged, when the expression breaks + /// the node rules (schema derivation, input validation, merge check). pub fn find_or_add_group( &mut self, expression: Operator, ) -> Result { - let _group_properties = derive_group_properties(self, &expression)?; - todo!("#1207 PR 2: hash-consing over groups") + let group_properties = derive_group_properties(self, &expression)?; + let hash = expression_hash(&expression); + let shareable = group_properties.shareable_when_equal; + if shareable { + if let Some(&group) = self + .groups_by_expression_hash + .get(&hash) + .and_then(|groups| { + groups.iter().find(|&&group| { + same_expression(&self.groups[group.0 as usize].expression, &expression) + }) + }) + { + return Ok(group); + } + } + let id = MemoGroupId( + u32::try_from(self.groups.len()).expect("a memo has at most u32::MAX groups"), + ); + for &input in expression.children() { + add_parent(&mut self.groups[input.0 as usize], id); + } + self.groups.push(MemoGroup { + expression, + group_properties, + parents: Vec::new(), + queries: Vec::new(), + candidates: Vec::new(), + }); + if shareable { + self.groups_by_expression_hash + .entry(hash) + .or_default() + .push(id); + } + Ok(id) } - /// Adds `candidate` to `group`'s candidates. Fails when its expression - /// breaks the node rules ([`derive_group_properties`]) or its output - /// schema or kind differs from the group's. + /// Adds `candidate` to `group`'s candidates and returns its id, unless + /// the group already has a candidate with the same expression + /// (`same_expression`) and logical properties: then that candidate's + /// id, and `candidate` is dropped. A new candidate makes `group` a parent + /// of each of its input groups, and `group`'s queries reach each input + /// group and every group below it. + /// + /// Fails, leaving the memo unchanged, when the candidate's expression + /// breaks the node rules a group's does ([`derive_group_properties`]), + /// or when its output schema or kind differs from the group's: then it + /// does not compute the group's result. pub fn add_candidate( &mut self, group: MemoGroupId, candidate: CandidateForMemoGroup, ) -> Result { - let _output = derive_group_properties(self, &candidate.expression.operator) + let output = derive_group_properties(self, &candidate.expression.operator) .map_err(CandidateError::Schema)?; - let _ = group; - todo!("#1207 PR 2") + let group_properties = &self.groups[group.0 as usize].group_properties; + if output.schema != group_properties.schema + || output.result_kind != group_properties.result_kind + { + return Err(CandidateError::OutputDiffersFromGroup { group }); + } + let candidates = &mut self.groups[group.0 as usize].candidates; + if let Some(index_in_group) = candidates + .iter() + .position(|known| same_expression_and_logical_properties(known, &candidate)) + { + return Ok(CandidateForMemoGroupId { + group, + index_in_group, + }); + } + let input_groups: Vec = candidate + .expression + .operator + .children() + .into_iter() + .copied() + .collect(); + candidates.push(candidate); + let index_in_group = candidates.len() - 1; + for &input in &input_groups { + add_parent(&mut self.groups[input.0 as usize], group); + } + let queries = self.groups[group.0 as usize].queries.clone(); + self.add_queries_below(input_groups, &queries); + Ok(CandidateForMemoGroupId { + group, + index_in_group, + }) + } + + /// Adds `queries` to each of `groups` and every group below them, through + /// the groups' expressions and their candidates' input groups. A group + /// that has every query already is not walked: every group below it has + /// them too. + fn add_queries_below(&mut self, groups: Vec, queries: &[usize]) { + let mut to_visit = groups; + while let Some(group) = to_visit.pop() { + let group = &mut self.groups[group.0 as usize]; + let mut added = false; + for &query in queries { + if let Err(position) = group.queries.binary_search(&query) { + group.queries.insert(position, query); + added = true; + } + } + if added { + to_visit.extend( + group + .expression + .children() + .into_iter() + .chain( + group + .candidates + .iter() + .flat_map(|candidate| candidate.expression.operator.children()), + ) + .copied(), + ); + } + } } /// The candidate `id`. @@ -146,6 +281,13 @@ impl Memo { } } +/// Lists `parent` among `group`'s parents, once. +fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { + if !group.parents.contains(&parent) { + group.parents.push(parent); + } +} + /// The groups some query's root reads, ascending: the groups of the /// `queries`' roots that are a query's result or a scalar query's operand. pub fn root_groups(queries: &[MemoQuery]) -> BTreeSet { @@ -162,3 +304,6 @@ pub fn queries_rooted_at(queries: &[MemoQuery], group: MemoGroupId) -> Vec Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +/// `SummaryAgg[KLL](input)`: no θ in the expression. +fn kll_state(input: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child: input, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +/// `SummaryEstimate[q](state)`. +fn quantile_readout(state: MemoGroupId, q: f64) -> Operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Quantile { q }, + }) +} + +/// An unordered `LIMIT 10` over `input`, whose copies may return +/// different rows. +fn unordered_limit(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: input, + }) +} + +/// `Project(input)` keeping its first column. +fn project_first_column(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![crate::ir::ProjectItem { + alias: None, + expr: crate::ir::ScalarExpr::Column(0), + }], + qualifier: None, + child: input, + }) +} + +/// A candidate computing `operator` with `parameters`. +fn candidate( + operator: Operator, + parameters: Option, +) -> CandidateForMemoGroup { + let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; + CandidateForMemoGroup { + // Built directly: the doc example's k = 200 and k = 345 are not + // in today's KLL list, which `LogicalProperties::new` checks. + logical_properties: LogicalProperties { parameters }, + expression: CandidateExpression { operator }, + input_requirements, + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + } +} + +/// A query rooted at `group`, without an accuracy target or $n_q$. +fn query(group: MemoGroupId) -> MemoQuery { + MemoQuery { + root: QueryRoot::Operator(group), + accuracy_target: None, + number_of_values_per_query_evaluation: None, + } +} + +/// KLL `k`. +fn kll_k(k: u32) -> Option { + Some(SketchParams::Kll { k }) +} + +/// An equal expression over the same input groups is found, not added. +#[test] +fn find_or_add_group_returns_the_group_of_an_equal_expression() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + assert_eq!(memo.find_or_add_group(scan("latency")).unwrap(), g1); + assert_eq!(memo.find_or_add_group(kll_state(g1)).unwrap(), g2); + assert_ne!(memo.find_or_add_group(scan("errors")).unwrap(), g1); + assert_eq!(memo.groups().len(), 3); + assert_eq!( + memo.groups()[g2.0 as usize].expression.children(), + vec![&g1] + ); +} + +/// The same operator over different input groups is a different group. +#[test] +fn find_or_add_group_separates_equal_operators_over_different_inputs() { + let mut memo = Memo::default(); + let latency = memo.find_or_add_group(scan("latency")).unwrap(); + let errors = memo.find_or_add_group(scan("errors")).unwrap(); + assert_ne!( + memo.find_or_add_group(kll_state(latency)).unwrap(), + memo.find_or_add_group(kll_state(errors)).unwrap() + ); +} + +/// An expression whose copies may return different rows (an unordered +/// `LIMIT`) gets a new group every time. +#[test] +fn find_or_add_group_never_returns_an_unshareable_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + assert_ne!( + memo.find_or_add_group(unordered_limit(g1)).unwrap(), + memo.find_or_add_group(unordered_limit(g1)).unwrap() + ); +} + +/// A new group is a parent of each of its input groups, listed once. +#[test] +fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + memo.find_or_add_group(kll_state(g1)).unwrap(); + assert_eq!(memo.groups()[g1.0 as usize].parent_groups(), &[g2]); + assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); +} + +/// KLL builds with k = 200 and k = 345 are two candidates of one group +/// G2: θ is a logical property, not part of the expression. +#[test] +fn kll_builds_with_k_200_and_k_345_share_one_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let k200_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k200 = memo + .add_candidate(k200_group, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + let k345_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k345 = memo + .add_candidate(k345_group, candidate(kll_state(g1), kll_k(345))) + .unwrap(); + assert_eq!(k200_group, k345_group); + assert_ne!(k200, k345); + assert_eq!(memo.groups()[k200_group.0 as usize].candidates.len(), 2); + assert_eq!( + memo.candidate(k345).logical_properties.parameters(), + kll_k(345).as_ref() + ); +} + +/// A candidate with the same expression and logical properties as one +/// in the group is not added again; its id is returned. +#[test] +fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let first = memo + .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + let again = memo + .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + assert_eq!(first, again); + assert_eq!( + first, + CandidateForMemoGroupId { + group: g2, + index_in_group: 0 + } + ); + let other_size = candidate(kll_state(g1), kll_k(345)); + assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); + assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); +} + +/// A new candidate makes its group a parent of each group it reads, and +/// the group's queries reach those groups and every group below them. +#[test] +fn add_candidate_maintains_parents_and_queries() { + // Query 0 is G3 = SummaryEstimate[q=0.95](G2) over G2 = KLL(G1). + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); + assert_eq!(memo.add_query(query(g3)), 0); + // A rule adds a state group G5 over another scan G4: no query + // reaches them yet. + let g4 = memo.find_or_add_group(scan("latency_copy")).unwrap(); + let g5 = memo.find_or_add_group(kll_state(g4)).unwrap(); + assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); + // G3 gains a candidate reading G5: G5 and G4 now serve query 0. + memo.add_candidate(g3, candidate(quantile_readout(g5, 0.95), None)) + .unwrap(); + assert_eq!(memo.groups()[g5.0 as usize].parent_groups(), &[g3]); + assert_eq!(memo.groups()[g4.0 as usize].parent_groups(), &[g5]); + assert_eq!(memo.groups()[g5.0 as usize].reached_by_queries(), &[0]); + assert_eq!(memo.groups()[g4.0 as usize].reached_by_queries(), &[0]); + // Query 1 is G6 = Project(G3): it reaches G3 and everything below + // it, through expressions and candidates, in ascending order. + let g6 = memo.find_or_add_group(project_first_column(g3)).unwrap(); + assert_eq!(memo.add_query(query(g6)), 1); + for group in [g1, g2, g3, g4, g5] { + assert_eq!( + memo.groups()[group.0 as usize].reached_by_queries(), + &[0, 1], + "group {group:?}" + ); + } + assert_eq!(memo.groups()[g3.0 as usize].parent_groups(), &[g6]); +} + +/// A candidate that computes another result than its group's (a +/// readout offered for a state group) is rejected and changes nothing. +#[test] +fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate(g2, candidate(quantile_readout(g2, 0.5), None)), + Err(CandidateError::OutputDiffersFromGroup { group }) if group == g2 + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); +} + +/// A candidate whose expression fails input validation (a readout of +/// raw rows instead of summary state) is rejected and changes nothing. +#[test] +fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); + let over_raw_rows = quantile_readout(g1, 0.95).map_children(|&group_id| MemoGroupAsInput { + memo: &memo, + group_id, + }); + assert!( + over_raw_rows.output_schema().is_ok(), + "fixture: the schema derives; only input validation rejects it" + ); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate(g3, candidate(quantile_readout(g1, 0.95), None)), + Err(CandidateError::Schema(_)) + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); +} + +/// An expression with no output schema (a group-by column out of range) +/// is an error, and no group is added. +#[test] +fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let out_of_range = Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![9]), + measures: vec![crate::ir::operator::agg_intent::AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child: g1, + }); + assert!(memo.find_or_add_group(out_of_range).is_err()); + assert_eq!(memo.groups().len(), 1); + assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); +} diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 249ca41cc..9c1a8f9fe 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -2,6 +2,7 @@ //! → candidate → candidate input → edge → Stage 1's output). mod candidate; +mod column_properties; mod edge; mod group; mod memo; diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 0aaf77cd4..544114faf 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -161,7 +161,7 @@ mod tests { /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last /// 5 minutes, each with its own expression as its one candidate. Built - /// through `find_or_add_group` and `add_candidate` (#1207 PR 2). + /// through `find_or_add_group` and `add_candidate`. fn memo() -> Memo { let mut memo = Memo::default(); let scan = memo @@ -310,7 +310,6 @@ mod tests { /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s /// serves both. #[test] - #[ignore = "filled in by #1207 PR 2 (#1230)"] fn delivers_on_a_chain_when_its_width_divides_the_context_times() { let candidates = candidates(memo(), vec![None; 5]); let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); @@ -349,7 +348,6 @@ mod tests { /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or /// another phase. Without a known grid it serves none. #[test] - #[ignore = "filled in by #1207 PR 2 (#1230)"] fn delivers_on_a_stored_result_on_its_group_grid() { let grid_60s = Some(EvaluationGrid { period_ms: 60_000, diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index bf56559c0..e4f12481b 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,8 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod memo_group_properties; +mod memo_summary_coverage; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs new file mode 100644 index 000000000..84930531b --- /dev/null +++ b/crates/types/tests/memo_group_properties.rs @@ -0,0 +1,491 @@ +//! `column_sign` and `depends_on_evaluation_time`, derived once per memo +//! group from its input groups (concepts doc, section 3, "Value sign" and +//! `GroupProperties`). Every example reads one of three sources: +//! +//! ```text +//! latency [ts, value, host] value declared non-negative +//! requests_total [ts, value, host] a counter: value non-negative +//! gauge [ts, value, host] nothing declared +//! ``` +use std::collections::BTreeMap; +use std::time::Duration; + +use asap_types::ir::operator::arguments::{ + AtModifier, GroupKeys, JoinKind, Reduction, Source, SourceColumn, TimeShift, +}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::{ColumnRef, ProjectItem}; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind, +}; +use asap_types::stages::{Memo, MemoGroupId, ValueSign}; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + +use ValueSign::{NonNegative, NonPositive, Unknown}; + +const TS: usize = 0; +const VALUE: usize = 1; +const HOST: usize = 2; +const MINUTE_MS: u64 = 60_000; + +/// The memo of every example: `latency.value` declared non-negative, and +/// `requests_total.value` as Stage 0 declares a counter's. +fn memo() -> Memo { + let non_negative = |source: &str| { + ( + SourceColumn { + source: source.into(), + column: "value".into(), + }, + NonNegative, + ) + }; + Memo::new(BTreeMap::from([ + non_negative("latency"), + non_negative("requests_total"), + ])) +} + +fn add(memo: &mut Memo, expression: Operator) -> MemoGroupId { + memo.find_or_add_group(expression).unwrap() +} + +fn signs(memo: &Memo, group: MemoGroupId) -> &[ValueSign] { + &memo.groups()[group.0 as usize].group_properties.column_sign +} + +fn depends(memo: &Memo, group: MemoGroupId) -> bool { + memo.groups()[group.0 as usize] + .group_properties + .depends_on_evaluation_time +} + +/// `Scan metric`: `[ts, value, host]`. +fn scan(metric: &str) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("host", DataType::Utf8, false), + ], + 0, + vec![], + ), + }) +} + +fn range(child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(MINUTE_MS), + kind: TimeRangeKind::Range, + child, + }) +} + +fn shift(at: Option, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 60_000, + at, + }, + child, + }) +} + +fn compare( + left: ScalarExpr, + op: CompareOpKind, + right: ScalarExpr, +) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(left), + op, + right: Box::new(right), + semantics: ExprSemantics::Sql, + } +} + +fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + }) +} + +/// `Project(ts, expr AS value, host)`. +fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator { + let item = |expr, alias: &str| ProjectItem { + alias: Some(alias.into()), + expr, + }; + Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + item(ScalarExpr::Column(TS), "ts"), + item(expr, "value"), + item(ScalarExpr::Column(HOST), "host"), + ], + qualifier: None, + child, + }) +} + +fn aggregate( + reduction: Reduction, + measure: AggIntent, + child: MemoGroupId, +) -> Operator { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures: vec![measure], + output_names: vec![], + filters: vec![], + having: None, + child, + }) +} + +fn by_host() -> Reduction { + Reduction::by(vec![HOST]) +} + +fn state(algorithm: SketchAlgorithm, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: FieldDataType::Sketch(algorithm), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query, + }) +} + +/// Rows of the per-operator table that keep or set a column's sign: +/// +/// | Group | Expression | `column_sign` | +/// |---|---|---| +/// | G1 | `Scan latency` | `[Unknown, NonNegative, Unknown]` (declared) | +/// | G2 | `Scan gauge` | `[Unknown, Unknown, Unknown]` (nothing declared) | +/// | G3 | `Filter(value > 1)(G1)` | G1's | +/// | G4 | `TimeRange[1m](G3)` | G1's | +/// | G5 | `TimeShift(1m)(G4)` | G1's | +/// | G6 | `Project(abs(value))(G2)` | value `NonNegative` | +/// | G7 | `Project(-value)(G1)` | value `NonPositive` (flipped) | +/// | G8 | `Project(-value)(G7)` | value `NonNegative` (flipped back) | +/// | G9 | `Project(value * 2)(G1)` | value `Unknown` (another expression) | +#[test] +fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let g2 = add(&mut memo, scan("gauge")); + let g3 = add( + &mut memo, + filter( + compare( + ScalarExpr::Column(VALUE), + CompareOpKind::Gt, + ScalarExpr::Literal(ScalarValue::Float64(1.0)), + ), + g1, + ), + ); + let g4 = add(&mut memo, range(g3)); + let g5 = add(&mut memo, shift(None, g4)); + let value = || Box::new(ScalarExpr::Column(VALUE)); + let negated = || ScalarExpr::Negative { + expr: value(), + semantics: ExprSemantics::Promql, + }; + let g6 = add( + &mut memo, + map_value( + ScalarExpr::FunctionCall { + name: "promql_abs".into(), + args: vec![ScalarExpr::Column(VALUE)], + }, + g2, + ), + ); + let g7 = add(&mut memo, map_value(negated(), g1)); + let g8 = add(&mut memo, map_value(negated(), g7)); + let g9 = add( + &mut memo, + map_value( + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: value(), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }, + g1, + ), + ); + let declared = [Unknown, NonNegative, Unknown]; + assert_eq!(signs(&memo, g1), declared); + assert_eq!(signs(&memo, g2), [Unknown; 3]); + for group in [g3, g4, g5] { + assert_eq!(signs(&memo, group), declared, "{group:?}"); + } + for (group, value) in [ + (g6, NonNegative), + (g7, NonPositive), + (g8, NonNegative), + (g9, Unknown), + ] { + assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{group:?}"); + } +} + +/// Aggregates: keys keep their sign, sum / max keep the value's, counts are +/// non-negative, and a rate is non-negative over a non-negative value only: +/// +/// | Group | Expression | `column_sign` | +/// |---|---|---| +/// | G2 | `sum by (host)(G1)`, G1 = `Scan latency` | `[Unknown, NonNegative]` | +/// | G3 | `max by (host)(G1)` | `[Unknown, NonNegative]` | +/// | G5 | `sum by (host)(G4)`, G4 = `Scan gauge` | `[Unknown, Unknown]` | +/// | G6 | `count by (host)(G4)` | `[Unknown, NonNegative]` | +/// | G7 | `count(distinct value) by (host)(G4)` | `[Unknown, NonNegative]` | +/// | G9 | `rate(G8)`, G8 = `requests_total[1m]` | value `NonNegative` | +/// | G11 | `rate(G10)`, G10 = `gauge[1m]` | value `Unknown` | +#[test] +fn aggregates_keep_their_keys_and_derive_their_values() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let sum = || AggIntent::Sum { col: Some(VALUE) }; + let g2 = add(&mut memo, aggregate(by_host(), sum(), g1)); + let g3 = add( + &mut memo, + aggregate(by_host(), AggIntent::Max { col: Some(VALUE) }, g1), + ); + let g4 = add(&mut memo, scan("gauge")); + let g5 = add(&mut memo, aggregate(by_host(), sum(), g4)); + let g6 = add(&mut memo, aggregate(by_host(), AggIntent::Count, g4)); + let g7 = add( + &mut memo, + aggregate(by_host(), AggIntent::Cardinality { cols: vec![VALUE] }, g4), + ); + let requests = add(&mut memo, scan("requests_total")); + let g8 = add(&mut memo, range(requests)); + let g9 = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, g8), + ); + let g10 = add(&mut memo, range(g4)); + let g11 = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, g10), + ); + for (group, value) in [ + (g2, NonNegative), + (g3, NonNegative), + (g5, Unknown), + (g6, NonNegative), + (g7, NonNegative), + ] { + assert_eq!(signs(&memo, group), [Unknown, value], "{group:?}"); + } + assert_eq!(signs(&memo, g9), [Unknown, NonNegative, Unknown]); + assert_eq!(signs(&memo, g11), [Unknown; 3]); +} + +/// A state records the sign of the values it is updated with, a merge +/// keeps the sign its inputs agree on, and a readout reads the state's, a +/// count-like statistic being non-negative whatever the values: +/// +/// | Group | Expression | `column_sign` | +/// |---|---|---| +/// | G3 | `SummaryAgg[KLL](G2)`, G2 = `latency[1m]` | state `NonNegative` | +/// | G6 | `SummaryAgg[KLL](G5)`, G5 = `(latency offset 1m)[1m]` | state `NonNegative` | +/// | G7 | `SummaryMerge(G3, G6)` | state `NonNegative` (agreed) | +/// | G8 | `SummaryEstimate[p99](G7)` | quantile `NonNegative` | +/// | G11 | `SummaryAgg[HLL](G10)`, G10 = `gauge[1m]` | state `Unknown` | +/// | G12 | `SummaryEstimate[cardinality](G11)` | cardinality `NonNegative` | +#[test] +fn states_record_their_input_sign_and_readouts_read_it() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let g2 = add(&mut memo, range(g1)); + let g3 = add(&mut memo, state(SketchAlgorithm::Kll, g2)); + let g4 = add(&mut memo, shift(None, g1)); + let g5 = add(&mut memo, range(g4)); + let g6 = add(&mut memo, state(SketchAlgorithm::Kll, g5)); + let g7 = add( + &mut memo, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![g3, g6], + group_by: None, + }), + ); + let g8 = add( + &mut memo, + readout(SketchStatistic::Quantile { q: 0.99 }, g7), + ); + let g9 = add(&mut memo, scan("gauge")); + let g10 = add(&mut memo, range(g9)); + let g11 = add(&mut memo, state(SketchAlgorithm::Hll, g10)); + let g12 = add(&mut memo, readout(SketchStatistic::Cardinality, g11)); + // [ts, state, host] + for group in [g3, g6, g7] { + assert_eq!( + signs(&memo, group), + [Unknown, NonNegative, Unknown], + "{group:?}" + ); + } + assert_eq!(signs(&memo, g8), [Unknown, NonNegative, Unknown]); + assert_eq!(signs(&memo, g11), [Unknown; 3]); + assert_eq!(signs(&memo, g12), [Unknown, NonNegative, Unknown]); +} + +/// The concepts doc's example: `topk(5, sum by (host)(gauge))` ranks values +/// of `Unknown` sign (Stage 2 then prunes the Count-Min heap builds), while +/// `topk(5, sum by (host)(rate(requests_total[1m])))` ranks `NonNegative` +/// ones. +#[test] +fn a_top_k_ranks_values_of_its_inputs_sign() { + let mut memo = memo(); + let top_5 = |memo: &mut Memo, child| { + add( + memo, + aggregate( + Reduction::Reduce(GroupKeys::by(vec![])), + AggIntent::TopK { k: 5 }, + child, + ), + ) + }; + let sum = AggIntent::Sum { col: Some(VALUE) }; + let gauge = add(&mut memo, scan("gauge")); + let gauge_sum = add(&mut memo, aggregate(by_host(), sum.clone(), gauge)); + let gauge_top = top_5(&mut memo, gauge_sum); + let requests = add(&mut memo, scan("requests_total")); + let ranged = add(&mut memo, range(requests)); + let rate = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, ranged), + ); + let rate_sum = add(&mut memo, aggregate(by_host(), sum, rate)); + let rate_top = top_5(&mut memo, rate_sum); + // [host, sum]: the ranked value is column 1 of the ranked rows. + assert_eq!(signs(&memo, gauge_sum), [Unknown, Unknown]); + assert_eq!(signs(&memo, rate_sum), [Unknown, NonNegative]); + let ranked = |group| { + let schema = &memo.groups()[group as usize].group_properties.schema; + let value = schema.column_id("topk_5").unwrap(); + signs(&memo, MemoGroupId(group))[value] + }; + assert_eq!(ranked(gauge_top.0), Unknown); + assert_eq!(ranked(rate_top.0), NonNegative); +} + +/// `depends_on_evaluation_time`: a `TimeRange` reads the window before the +/// evaluation time, `now()` reads it directly, and everything above them +/// depends on it, unless `@ T` pins the subtree; `@ end()` does not pin it +/// to a fixed time. +/// +/// | Group | Expression | Depends | +/// |---|---|---| +/// | G1 | `Scan latency` | no | +/// | G2 | `sum by (host)(G1)` | no | +/// | G3 | `latency[1m]` | yes | +/// | G4 | `SummaryAgg[KLL](G3)` | yes | +/// | G5 | `TimeShift(@ T, offset 1m)(G3)` | no | +/// | G6 | `SummaryAgg[KLL](G5)` | no | +/// | G7 | `TimeShift(@ end(), offset 1m)(G3)` | yes | +/// | G8 | `Filter(ts > now())(G1)` | yes | +#[test] +fn a_result_depends_on_the_evaluation_time_under_a_time_range_or_now() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let g2 = add( + &mut memo, + aggregate(by_host(), AggIntent::Sum { col: Some(VALUE) }, g1), + ); + let g3 = add(&mut memo, range(g1)); + let g4 = add(&mut memo, state(SketchAlgorithm::Kll, g3)); + let g5 = add( + &mut memo, + shift(Some(AtModifier::Timestamp(1_700_000_000_000)), g3), + ); + let g6 = add(&mut memo, state(SketchAlgorithm::Kll, g5)); + let g7 = add(&mut memo, shift(Some(AtModifier::End), g3)); + let g8 = add( + &mut memo, + filter( + compare( + ScalarExpr::Column(TS), + CompareOpKind::Gt, + ScalarExpr::CurrentTimestamp, + ), + g1, + ), + ); + for (group, expected) in [ + (g1, false), + (g2, false), + (g3, true), + (g4, true), + (g5, false), + (g6, false), + (g7, true), + (g8, true), + ] { + assert_eq!(depends(&memo, group), expected, "{group:?}"); + } +} + +/// `reads_only_time_series`, derived once per group: a PromQL `Scan` of a +/// time series reads one, and so does a `TimeRange` over it; a SQL table +/// `Scan` does not, nor a `Join` reading it (a SQL join reads relations +/// only, so a time series cannot be one of its inputs). +#[test] +fn a_group_reads_only_time_series_when_every_scan_below_reads_one() { + let mut memo = memo(); + let series = add(&mut memo, scan("latency")); + let window = add(&mut memo, range(series)); + let table = |name: &str| { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: name.into(), + }, + predicates: vec![], + schema: Schema::new(vec![ + Field::plain("host", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + }) + }; + let requests = add(&mut memo, table("requests")); + let hosts = add(&mut memo, table("hosts")); + let join = add( + &mut memo, + Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: requests, + right: hosts, + }), + ); + let reads = |group: MemoGroupId| { + memo.groups()[group.0 as usize] + .group_properties + .reads_only_time_series + }; + assert!(reads(series)); + assert!(reads(window)); + assert!(!reads(requests)); + assert!(!reads(join)); +} diff --git a/crates/types/tests/memo_summary_coverage.rs b/crates/types/tests/memo_summary_coverage.rs new file mode 100644 index 000000000..e2199edb1 --- /dev/null +++ b/crates/types/tests/memo_summary_coverage.rs @@ -0,0 +1,681 @@ +//! Group properties and aggregate coverage, derived once per memo group +//! from its input groups (`docs/design_docs/proposals/asap-primitive-schema.md` +//! §4.2.2). The memo of the worked example: +//! +//! ```text +//! G1 Scan latency +//! G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) +//! G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) +//! G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) +//! ``` +use std::ops::Bound; +use std::time::Duration; + +use asap_types::ir::operator::arguments::{AtModifier, GroupKeys, Reduction, Source, TimeShift}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::summary_coverage::{ + ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, SummaryCoverage, +}; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, + SchemaDerivationError, TimeRangeKind, +}; +use asap_types::stages::{ + CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, + MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::{ + ExactKind, MergeInputRule, SketchAlgorithm, SketchStatistic, SummaryUpdate, +}; +use asap_types::test_support::definition_id; + +const TS: usize = 0; +const JOB: usize = 2; +const A: usize = 3; +const B: usize = 4; +const MINUTE_MS: i64 = 60_000; +const HOUR_MS: i64 = 60 * MINUTE_MS; + +/// `Scan latency{predicates}`: `[ts, value, job, a, b]`. +fn scan(predicates: Vec>) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: predicates.into_iter().map(Predicate).collect(), + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, false), + Field::plain("a", DataType::Float64, false), + Field::plain("b", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +fn job_is(job: &str) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(JOB)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(job.into()))), + semantics: ExprSemantics::Sql, + } +} + +/// `a + b > 10`: it spans two columns, so it is no selection. +fn a_plus_b_above_10() -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Add, + left: Box::new(ScalarExpr::Column(A)), + right: Box::new(ScalarExpr::Column(B)), + semantics: ExprSemantics::Sql, + }), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + } +} + +fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + }) +} + +/// `TimeShift(offset_ms, at)`: PromQL `offset` and `@`. +fn shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { offset_ms, at }, + child, + }) +} + +fn range(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(width_ms as u64), + kind: TimeRangeKind::Range, + child, + }) +} + +/// `SummaryAgg[summary_type]` of the sample value per series. +fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +fn kll_summary_type() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) +} + +/// `SummaryAgg[KLL]` of the sample value per series. +fn kll(child: MemoGroupId) -> Operator { + state(kll_summary_type(), child) +} + +/// `SummaryTumblingWindowChain[KLL, width_ms]` of the sample value per series. +fn kll_chain(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + summary_type: kll_summary_type(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: width_ms as u64, + }) +} + +fn merge(children: Vec) -> Operator { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + }) +} + +/// KLL over the last minute of `G1` filtered by `pred`. +fn kll_of_last_minute_where( + memo: &mut Memo, + g1: MemoGroupId, + pred: ScalarExpr, +) -> MemoGroupId { + let filtered = memo.find_or_add_group(filter(pred, g1)).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, filtered)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() +} + +fn coverage(memo: &Memo, group: MemoGroupId) -> &SummaryCoverage { + memo.groups()[group.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state") +} + +/// `KLL` as an unlifted operator: its main input is `None`. +fn kll_unlifted() -> Operator> { + kll(MemoGroupId(0)).map_children(|_| None) +} + +fn jobs(values: &[&str]) -> Constraint { + Constraint::In( + values + .iter() + .map(|value| ScalarValue::Utf8((*value).into())) + .collect(), + ) +} + +/// `(t + lower_ms, t + upper_ms]` on the timestamp column, t the evaluation +/// time: left-open, right-closed like a PromQL range. +fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { + Constraint::Interval { + lower: Bound::Excluded(IntervalConstraintLimit::EvaluationTimePlusMs(lower_ms)), + upper: Bound::Included(IntervalConstraintLimit::EvaluationTimePlusMs(upper_ms)), + } +} + +fn last_minute() -> Constraint { + relative(-MINUTE_MS, 0) +} + +fn ts() -> ColumnIdentity { + ColumnIdentity::new(None, "ts") +} + +/// The constraint on the timestamp column of `group`'s one selection box. +fn time_of(memo: &Memo, group: MemoGroupId) -> Constraint { + let [selection] = coverage(memo, group).selection() else { + panic!("one box: {:?}", coverage(memo, group)) + }; + selection.columns()[&ts()].clone() +} + +/// A group's properties come from its operator and its input groups' +/// properties: the scan's schema, the state's kind and state column, the +/// readout's value column, and whether equal copies may be one group. +#[test] +fn group_properties_come_from_input_groups() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let readout = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Quantile { q: 0.99 }, + })) + .unwrap(); + let limit = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { + n: Some(1), + offset: 0, + partition_by: GroupKeys::default(), + child: g1, + })) + .unwrap(); + let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; + assert_eq!(properties(g1).schema.fields.len(), 5); + assert_eq!(properties(g1).aggregate_coverage, None); + assert_eq!(properties(g4).result_kind, OperatorResultKind::State); + assert!(properties(g4) + .schema + .fields + .iter() + .any(|field| field.name == "state" && !field.is_plain())); + assert!(properties(g4).aggregate_coverage.is_some()); + assert_eq!( + properties(readout).result_kind, + OperatorResultKind::InstantVector + ); + assert!(properties(readout) + .schema + .fields + .iter() + .any(|field| field.name == "quantile" + && field.dtype == FieldDataType::Plain(DataType::Float64))); + assert!(properties(g4).shareable_when_equal); + assert!(!properties(limit).shareable_when_equal); +} + +/// G4: `job = "api"` and the minute are lifted; the definition is the KLL +/// over G1. +#[test] +fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!( + coverage.definition().unlifted_operators, + vec![kll_unlifted()] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert_eq!(selection.columns()[&ts()], last_minute()); +} + +/// G7 has G4's definition and id and a disjoint selection, so +/// `SummaryMerge(G4, G7)` is a valid group covering job ∈ {api, web}. +#[test] +fn states_with_equal_definitions_and_disjoint_selections_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g7 = kll_of_last_minute_where(&mut memo, g1, job_is("web")); + assert_eq!( + coverage(&memo, g4).definition(), + coverage(&memo, g7).definition() + ); + assert_eq!( + definition_id(coverage(&memo, g4)), + definition_id(coverage(&memo, g7)) + ); + let merged = memo.find_or_add_group(merge(vec![g4, g7])).unwrap(); + let coverage = coverage(&memo, merged); + assert_eq!(coverage.definition().input_group, Some(g1)); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// Two states that select the same rows do not merge: KLL counts a row +/// once per input. +#[test] +fn states_with_overlapping_selections_do_not_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let groups = memo.groups().len(); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g4])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::Disjoint + } + )) + )); + assert_eq!(memo.groups().len(), groups); +} + +/// G10: `a + b > 10` spans two columns, so it stays as a `Filter` in the +/// definition; G10 does not merge with G4 although both select job = api. +#[test] +fn a_filter_that_is_no_selection_stays_unlifted() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g10 = kll_of_last_minute_where( + &mut memo, + g1, + ScalarExpr::BoolAnd(vec![job_is("api"), a_plus_b_above_10()]), + ); + let coverage_of_g10 = coverage(&memo, g10); + assert_eq!(coverage_of_g10.definition().input_group, Some(g1)); + assert_eq!( + coverage_of_g10.definition().unlifted_operators, + vec![ + kll_unlifted(), + filter(a_plus_b_above_10(), MemoGroupId(0)).map_children(|_| None), + ] + ); + assert_eq!( + coverage_of_g10.selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g10])), + Err(SchemaDerivationError::Coverage( + CoverageError::DefinitionMismatch + )) + )); +} + +/// `a > (SELECT threshold)`: a scalar subquery reading group `threshold`. +fn a_above(threshold: MemoGroupId) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(A)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::ScalarSubquery(threshold)), + semantics: ExprSemantics::Sql, + } +} + +/// A residual `Filter` that reads a scalar subquery names the subquery's +/// group (`Some`) next to its main input (`None`). Two states with that +/// equal definition and disjoint selections merge. +#[test] +fn states_whose_definitions_read_the_same_subquery_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let threshold = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Values { + rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(10.0))]], + schema: Schema::new(vec![Field::plain("threshold", DataType::Float64, false)]), + })) + .unwrap(); + let state_of_job = |memo: &mut Memo, job: &str| { + let pred = ScalarExpr::BoolAnd(vec![job_is(job), a_above(threshold)]); + kll_of_last_minute_where(memo, g1, pred) + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input_group, Some(g1)); + assert_eq!( + definition.unlifted_operators, + vec![ + kll_unlifted(), + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(a_above(threshold).map_operator_refs(&mut |group| Some(*group))), + child: None, + }), + ] + ); + assert_eq!(definition, coverage(&memo, web).definition()); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// PromQL label matchers are the scan's predicates: they are lifted too, so +/// `latency{job="api"}` and `latency{job="web"}` states merge. The scan, +/// without them, is the last unlifted operator. +#[test] +fn scan_predicates_lift_and_their_states_merge() { + let mut memo = Memo::default(); + let state_of_job = |memo: &mut Memo, job: &str| { + let series = memo.find_or_add_group(scan(vec![job_is(job)])).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input_group, None); + assert_eq!( + definition.unlifted_operators, + vec![kll_unlifted(), scan(vec![]).map_children(|_| None)] + ); + assert_eq!( + coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// A chain has the definition of a build over its input and leaves the +/// timestamp column unconstrained. `TimeRange` and `TimeShift` over it +/// select windows as over raw rows, so the merge over that selection covers +/// what the direct build does, and `add_candidate` accepts the merge in the +/// direct build's group: +/// +/// ```text +/// G1 Scan latency +/// G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) (the query) +/// G5 Chain[KLL, 1m](G1); G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) +/// G4 candidate: SummaryMerge(G7) +/// ``` +#[test] +fn a_merge_over_a_chains_windows_covers_the_direct_build() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(shift(HOUR_MS, None, g1)).unwrap(); + let g3 = memo.find_or_add_group(range(5 * MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(kll(g3)).unwrap(); + let g5 = memo.find_or_add_group(kll_chain(MINUTE_MS, g1)).unwrap(); + let g6 = memo.find_or_add_group(shift(HOUR_MS, None, g5)).unwrap(); + let g7 = memo.find_or_add_group(range(5 * MINUTE_MS, g6)).unwrap(); + + let whole_chain = coverage(&memo, g5); + assert_eq!(whole_chain.definition(), coverage(&memo, g4).definition()); + assert!(whole_chain.selection()[0].columns().is_empty()); + let five_minutes_an_hour_ago = relative(-HOUR_MS - 5 * MINUTE_MS, -HOUR_MS); + assert_eq!(time_of(&memo, g7), five_minutes_an_hour_ago); + assert_eq!(coverage(&memo, g7), coverage(&memo, g4)); + + let expression = CandidateExpression { + operator: merge(vec![g7]), + }; + let merge_of_windows = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(g4, merge_of_windows).unwrap(); + let merged = memo.find_or_add_group(merge(vec![g7])).unwrap(); + assert_eq!(coverage(&memo, merged), coverage(&memo, g4)); +} + +/// The range and every offset lift into one interval relative to the +/// evaluation time t; `@ T` fixes the interval in absolute time: +/// +/// | Group | Expression | Timestamp column | +/// |---|---|---| +/// | G2 | `TimeRange[5m](G1)` (`m[5m]`) | `(t − 5m, t]` | +/// | G4 | `TimeRange[5m](TimeShift(1h)(G1))` (`m[5m] offset 1h`) | `(t − 65m, t − 60m]` | +/// | G6 | `TimeRange[5m](TimeShift(@ 1700000000)(G1))` | `(1699999700000, 1700000000000]` | +/// +/// each under a `SummaryAgg[KLL]`. +#[test] +fn time_range_and_shifts_lift_into_one_interval_on_the_timestamp_column() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let plain = kll_over_last_5m(&mut memo, g1); + let offset = kll_over_last_5m_of(&mut memo, shift(HOUR_MS, None, g1)); + let at_t = kll_over_last_5m_of( + &mut memo, + shift(0, Some(AtModifier::Timestamp(1_700_000_000_000)), g1), + ); + assert_eq!(time_of(&memo, plain), relative(-5 * MINUTE_MS, 0)); + assert_eq!(time_of(&memo, offset), relative(-3_900_000, -3_600_000)); + let timestamp = |ms| IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(ms)); + assert_eq!( + time_of(&memo, at_t), + Constraint::Interval { + lower: Bound::Excluded(timestamp(1_699_999_700_000)), + upper: Bound::Included(timestamp(1_700_000_000_000)), + } + ); + for state in [offset, at_t] { + assert_eq!( + coverage(&memo, state).definition(), + coverage(&memo, plain).definition() + ); + } +} + +/// `SummaryAgg[KLL](TimeRange[5m](G1))`: `m[5m]`. +fn kll_over_last_5m(memo: &mut Memo, input: MemoGroupId) -> MemoGroupId { + let ranged = memo.find_or_add_group(range(5 * MINUTE_MS, input)).unwrap(); + memo.find_or_add_group(kll(ranged)).unwrap() +} + +/// [`kll_over_last_5m`] over the group of `expression`. +fn kll_over_last_5m_of(memo: &mut Memo, expression: Operator) -> MemoGroupId { + let input = memo.find_or_add_group(expression).unwrap(); + kll_over_last_5m(memo, input) +} + +/// `@ end()` and `@ start()` are lifted above the subtree they anchor: +/// the front end lowers `m[5m] @ end()` as `TimeShift{@ end()}(G2)` over +/// G2 = `m[5m]`, the group an instant query's `m[5m]` reads too. The +/// `TimeShift` pins G2 to one end of the request, so it is not lifted: it +/// is the input group of the state over it, whose selection is empty. +/// +/// ```text +/// G1 Scan latency; G2 TimeRange[5m](G1); G3 SummaryAgg[KLL](G2) (instant query) +/// G4 TimeShift(@ end())(G2); G5 SummaryAgg[KLL](G4) (range query) +/// ``` +#[test] +fn at_end_and_at_start_share_the_subtree_they_anchor_and_stay_unlifted() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let g3 = memo.find_or_add_group(kll(g2)).unwrap(); + assert_eq!(time_of(&memo, g3), relative(-5 * MINUTE_MS, 0)); + for at in [AtModifier::End, AtModifier::Start] { + let pinned = memo.find_or_add_group(shift(0, Some(at), g2)).unwrap(); + assert_eq!( + memo.groups()[pinned.0 as usize].expression.children(), + [&g2] + ); + let state = memo.find_or_add_group(kll(pinned)).unwrap(); + let coverage = coverage(&memo, state); + assert_eq!(coverage.definition().input_group, Some(pinned)); + assert_eq!(coverage.definition().unlifted_operators, [kll_unlifted()]); + assert!(coverage.selection()[0].columns().is_empty()); + } +} + +/// `ts >= ` (a SQL timestamp filter) and `value > 10` lift as +/// intervals between fixed values: `G3 SummaryAgg[KLL](G2)`, `G2 +/// Filter(ts >= 2026-11-01 ∧ value > 10)(G1)`. +#[test] +fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + let compare = |column, op, value| ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(value)), + semantics: ExprSemantics::Sql, + }; + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo + .find_or_add_group(filter( + ScalarExpr::BoolAnd(vec![ + compare(TS, CompareOpKind::Ge, ScalarValue::Timestamp(NOV_1_2026_MS)), + compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), + ]), + g1, + )) + .unwrap(); + let g3 = memo.find_or_add_group(kll(g2)).unwrap(); + let fixed = |value| Bound::Included(IntervalConstraintLimit::FixedValue(value)); + assert_eq!( + time_of(&memo, g3), + Constraint::Interval { + lower: fixed(ScalarValue::Timestamp(NOV_1_2026_MS)), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).selection()[0].columns()[&ColumnIdentity::new(None, "value")], + Constraint::Interval { + lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( + 10.0 + ))), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).definition().unlifted_operators, + vec![kll_unlifted()] + ); +} + +/// Counter states (`Increase`) merge only over windows disjoint in time: +/// the last 5 minutes and the 5 before them merge into the last 10; two +/// copies of the last 5 minutes do not. A KLL over the last minute and one +/// over the minute before it are disjoint too. +/// +/// ```text +/// G2 TimeRange[5m](G1); G3 SummaryAgg[Increase](G2) +/// G4 TimeShift(5m)(G1); G5 TimeRange[5m](G4); G6 SummaryAgg[Increase](G5) +/// G7 SummaryMerge(G3, G6) +/// ``` +#[test] +fn counter_states_merge_over_windows_disjoint_in_time() { + let increase = || FieldDataType::exact(ExactKind::Increase); + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let g3 = memo.find_or_add_group(state(increase(), g2)).unwrap(); + let g4 = memo + .find_or_add_group(shift(5 * MINUTE_MS, None, g1)) + .unwrap(); + let g5 = memo.find_or_add_group(range(5 * MINUTE_MS, g4)).unwrap(); + let g6 = memo.find_or_add_group(state(increase(), g5)).unwrap(); + let g7 = memo.find_or_add_group(merge(vec![g3, g6])).unwrap(); + assert_eq!(time_of(&memo, g7), relative(-10 * MINUTE_MS, 0)); + assert!(matches!( + memo.find_or_add_group(merge(vec![g3, g3])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::DisjointInTime + } + )) + )); + let g8 = memo.find_or_add_group(kll(g2)).unwrap(); + let g9 = memo.find_or_add_group(kll(g5)).unwrap(); + let g10 = memo.find_or_add_group(merge(vec![g8, g9])).unwrap(); + assert_eq!(time_of(&memo, g10), relative(-10 * MINUTE_MS, 0)); +} + +/// An exact `Aggregate` gets coverage like a summary state: `G4 +/// Aggregate[sum(value) by job](G3)` over the last minute of `job = "api"` +/// has the `Aggregate` as its definition and the job and minute as its +/// selection. +#[test] +fn an_exact_aggregate_has_coverage() { + let sum_by_job = |child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![JOB]), + measures: vec![AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }; + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(filter(job_is("api"), g1)).unwrap(); + let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(sum_by_job(g3)).unwrap(); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!( + coverage.definition().unlifted_operators, + vec![sum_by_job(MemoGroupId(0)).map_children(|_| None)] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert_eq!(selection.columns()[&ts()], last_minute()); +} diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index e0a25e1a3..156e0457f 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -114,10 +114,10 @@ A node in the physical data will represent the data or summary instance, so a no | Why not put it in the schema? | States worth merging cover different data but must have the same schema | below | | What is it based on? | Goldstein & Larson view matching (SIGMOD 2001), explained with an example | §4.1 | | What does it store? | `definition` (what is computed) + `selection` (which rows were taken) | §4.2.1 | -| How is it computed? | by the planner, from the sub-DAG the node covers, in one walk that yields both the definition and the selection | §4.2.2 | -| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; cached per node | §4.2.3 | +| How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | +| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `OperatorNode::coverage()`, `SummaryCoverage::derive` | §6.1, §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, `SummaryCoverage::derive_from_inputs` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -237,70 +237,119 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -The planner computes the coverage of a node from the sub-DAG the node covers, not from a declaration. One walk down the sub-DAG produces both parts: every filter condition either moves into `selection` or stays in `definition`. The definition is described first, then how the planner decides which conditions move into the selection. +Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), `SummaryCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::aggregate_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). -**Worked example.** p99 latency per job over table `t` (columns `job`, `region`, `latency`): +```rust +/// What a summary state computes, with every lifted selection removed. +pub struct SummaryStateDefinition { + /// The group the unlifted operators read: the first input under the + /// SummaryAgg or Aggregate that is not Filter / range TimeRange / + /// TimeShift / direct-column Project. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator. + pub input_group: Option, + /// The operators not lifted into the selection, listed top down from the + /// SummaryAgg, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or input_group); + /// `Some(g)` is another input group it reads, such as the group a scalar + /// subquery in a Filter reads. + pub unlifted_operators: Vec>>, +} -```sql -SELECT job, - approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) -FROM (SELECT job, region AS r, latency - FROM t - WHERE region = 'us' AND latency * 2 > 10) -GROUP BY job; +pub struct SummaryCoverage { + pub(crate) definition: SummaryStateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. + pub(crate) selection: Vec, +} + +impl SummaryCoverage { + pub(crate) fn derive_from_inputs( + operator: &Operator>, + ) -> Result; + /// A hash of the definition: equal definitions of one memo get equal ids. + pub(crate) fn definition_id(&self) -> String; +} ``` -The SQL frontend lowers it as follows: `FILTER (WHERE …)` becomes the aggregate's filter, `region AS r` becomes a `Project`, and the `WHERE` conditions are folded into the `Scan`'s predicates. With an error target, the planner answers the quantile from a KLL `SummaryAgg`, read out by a `SummaryEstimate`. The sub-DAG under that `SummaryEstimate`: +**Worked example.** Three queries over the series `latency` (labels `job`, `a`, `b`), each a KLL state over the last minute: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job, filter: latency < 100 - ▲ - │ - [ Project ] job, region AS r, latency - ▲ - │ - [ Scan t ] predicates: region = 'us' AND latency * 2 > 10 +G1 Scan latency +G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) ← Q1 +G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) ← Q2 +G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) ← Q3 ``` -##### The definition +**Case 1, G4: everything is lifted.** The walk starts at G4's `SummaryAgg` and reads each input group's `expression`: + +| Step | Group | Expression | What the walk does | +|---|---|---|---| +| 1 | G4 | `SummaryAgg[KLL](G3)` | the first unlifted operator, with no filter of its own | +| 2 | G3 | `TimeRange[1m](G2)` | the one range window: lifted as `ts ∈ (t − 60 s, t]` (t the evaluation time) | +| 3 | G2 | `Filter(job = "api")(G1)` | `job = "api"` is a value set on one column: lifted, nothing of the `Filter` remains | +| 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input_group` | + +```text +G4.aggregate_coverage = SummaryCoverage { + definition: SummaryStateDefinition { + input_group: Some(G1), + unlifted_operators: [SummaryAgg[KLL]], + }, + selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], +} +``` + +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. + +**Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: +```text +G10.aggregate_coverage = SummaryCoverage { + definition: SummaryStateDefinition { + input_group: Some(G1), + unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], + }, + selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], +} +``` -The `definition` is the `SummaryAgg` together with its sub-DAG, with every condition that moves into `selection` (below) taken out. Everything else stays exactly as it is: +Its definition differs from G4's, so G10 cannot merge with G4, even though both select `job = api`. + +**The fields.** + +- `input_group`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). +- `unlifted_operators`: what the state computes over `input_group`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. +- `selection`: which output rows of that computation the state took, as a union of boxes. +- `definition_id()`: a hash of the definition, the key a summary store files states under together with a concrete selection. + +##### The definition -| In the sub-DAG | In the `definition` | +The `definition` is what remains of the operators the walk passed, with every condition that moves into `selection` (below) taken out: + +| On the walk | In `unlifted_operators` | |---|---| | a `Filter` whose conditions all move into `selection` | removed | -| a `Filter` with some conditions that stay | kept, with only the conditions that stay | -| `Scan.predicates` and `SummaryAgg.filter` | trimmed the same way | -| the one range `TimeRange`, and any `TimeShift`s, when they become relative time | removed | -| a `TimeShift` with no range `TimeRange`, or two or more range `TimeRange`s | unchanged: no time is taken out | -| any other operator | unchanged | - -So the `definition` holds what the state computes: the computation `C` with its remaining conditions, the summary family and its parameters, the input column, and the grouping `G`. +| a `Filter` with some conditions that stay | kept, with only the conditions that stay; a scalar subquery among them names its group | +| `SummaryAgg.filter` | trimmed the same way | +| the one range `TimeRange`, and every `TimeShift` (with or without `@`), when they become one interval on the timestamp column | removed | +| a `TimeShift` with no range `TimeRange`, two or more range `TimeRange`s, two `@`s, or an interval that does not combine with a timestamp condition already lifted | unchanged: no time is taken out | +| a direct-column `Project` | unchanged | +| a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input_group` is `None` | +| any other operator (the first group the walk cannot pass) | not in the list: it is the `input_group` | -In the worked example, `latency < 100` and `region = 'us'` move into `selection` (below), and `latency * 2 > 10` stays: +An exact `Aggregate` is the first unlifted operator, kept whole (its per-measure filters and `HAVING` read its input or output, not the rows it reads). A `SummaryTumblingWindowChain[w](X)` has the definition of a build over X, and its selection leaves the timestamp column unconstrained: a chain has no range. A `TimeRange` or `TimeShift` over a chain (a group whose output is the chain's state with a time index) has the chain's coverage with the interval they select on the timestamp column of X, computed as over raw rows. A `SummaryMerge` over that group reads it, so `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` covers exactly what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does, and `Memo::add_candidate` accepts the merge as a candidate of the direct build's group: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job ← filter removed - ▲ - │ - [ Project ] job, region AS r, latency ← unchanged - ▲ - │ - [ Scan t ] predicates: latency * 2 > 10 ← region = 'us' removed +G1 Scan latency +G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) ts ∈ (t − 65m, t − 60m] +G5 Chain[KLL, 1m](G1) ts unconstrained +G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) ts ∈ (t − 65m, t − 60m] +G4 candidate: SummaryMerge(G7) the same coverage as G4 ``` -**When two definitions are equal.** Merging (§5.6) requires equal definitions. +A rollup (`SummaryMerge` with `group_by`) has its input's definition with the `SummaryAgg` grouped by the rollup's keys. -- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. -- `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. -- States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. +**When two definitions are equal.** Merging (§5.6) requires equal definitions: the same `input_group` and the same `unlifted_operators`, compared as values. A definition with a scalar subquery compares like any other: two states whose `Filter(a > (SELECT threshold))` read the same group are equal there. Timing is not part of a group, so a tumbling window built at ingestion time can merge with one built at query time. States over different tables never merge: a KLL over `m1` and one over `m2` read different groups. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. ##### The selection @@ -309,7 +358,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **Steps.** -1. **Collect the conditions.** Go down the sub-DAG from the `SummaryAgg` and collect every filter condition: from `Filter` nodes, from `Scan.predicates`, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. +1. **Collect the conditions.** Go down the input groups from the `SummaryAgg` and collect every filter condition: from `Filter` groups, from the `Scan.predicates` where the walk stops, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. 2. **Ask two questions about each condition:** - **Rule 1: would it pick the same rows if it were moved to just below the `SummaryAgg`?** `region = 'us'` below a `Project` that only renames columns: yes. `value > 5` below `rate`: no, because it filters the raw samples that `rate` reads, which changes the rate values. - **Rule 2: is it a simple condition on one column?** That is, a value set such as `region IN ('us', 'eu')`, or a range such as `latency < 100`. `latency * 2 > 10` is not: it is on an expression. @@ -318,7 +367,16 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` - If either answer is no: leave it in the sub-DAG, so it is part of `definition`. The paper calls such conditions *residuals*. - One more case stays in `definition`: a condition on a column that already has a lifted condition it cannot be combined with (a forbidden-value set and a range, or values of different types). Conditions are taken in walk order: the `SummaryAgg.filter`, then the `Filter`s from top to bottom, then `Scan.predicates`. -In the worked example: +A SQL example, p99 latency per job over table `t` (columns `job`, `region`, `latency`), whose `WHERE` conditions the SQL front end folds into the `Scan`'s predicates and whose `region AS r` is a `Project`: + +```sql +SELECT job, + approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) +FROM (SELECT job, region AS r, latency + FROM t + WHERE region = 'us' AND latency * 2 > 10) +GROUP BY job; +``` | Condition | Found at | Rule 1: same rows at the `SummaryAgg`? | Rule 2: simple? | Result | |---|---|---|---|---| @@ -333,11 +391,11 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl | Operator it must pass | Can it pass? | Why | Example | |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | -| `TimeRange` (range) or `TimeShift` (without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | +| `TimeRange` (range) or `TimeShift` (with or without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | | `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | -| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit`, a `TimeShift` with `@` | no | | | +| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit` | no | | | A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. How a PromQL comparison is lowered decides whether it is such a `Filter`: @@ -377,18 +435,21 @@ Any other shape stays in `definition`: - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. -**Time.** Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`. Its KLL state (one per series) has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = time `(−3m, −2m]`, the minute that ended 2 minutes before evaluation. There are two kinds of time: +**Time.** The timestamp column is a column like any other: a selection constrains it with an interval whose ends are `IntervalConstraintLimit`s, `EvaluationTimePlusMs(ms)` (relative to the evaluation time t) or `FixedValue(Timestamp(ms since the epoch))` (absolute). The walk turns the one range `TimeRange(W)` and the `TimeShift`s on its path, whose offsets sum to o, into `ts ∈ (anchor − o − W, anchor − o]` on the timestamp column of the `TimeRange`'s input. Like a PromQL range, which `m[r]` evaluated at t reads as (t − r, t], the interval is left-open and right-closed. The anchor is t, or T for `@ T`. `@ end()` and `@ start()` pin a subtree to one end of the request: the PromQL front end lowers the subtree X without them and puts `TimeShift { at: End | Start }(X)` above it, so X is the group of the same expression without `@`. That `TimeShift` is not lifted: the walk stops at it, and it is the definition's `input_group`. Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`; its KLL state has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = `ts ∈ (t − 3m, t − 2m]`. -| Kind | Comes from | Example | +| Query fragment | `lower` | `upper` | |---|---|---| -| **Relative** to the evaluation time | a range `TimeRange(w)` over a `TimeShift(s)` → `(−(s+w), −s]`; a SQL trailing window `ts >= now() - INTERVAL 'w'` lowers to `TimeRange(w)` | `TimeRange(1m)` alone → `(−1m, 0]`; over `TimeShift(1m)` → `(−2m, −1m]` | -| **Absolute** (later) | an interval on the timestamp column | `ts >= t0 AND ts < t1` → `ts ∈ [t0, t1)` | +| PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | +| PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | +| PromQL `m[5m] @ end()` or `@ start()` | the subtree `m[5m]` is the group of `m[5m]`; the `TimeShift { at: End \| Start }` above it is not lifted (it is `input_group`) | | +| PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | +| SQL `WHERE ts >= now() - INTERVAL '5 minutes'` | lowers to `TimeRange(5m)`: as `m[5m]` | | +| SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(1_793_491_200_000)))` | `Unbounded` | -- PromQL windows exclude their start, so relative windows are open on the left. A SQL trailing window takes the same timing, `>=` included. -- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) builds its tumbling windows this way: a 3-minute window as tumbling windows `(−1m, 0]`, `(−2m, −1m]`, `(−3m, −2m]`, so tumbling window times are derived, not declared (#601). +- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) reads a longer range from a chain: `TimeRange(3m)` over a 1-minute chain selects the windows `(t − 1m, t]`, `(t − 2m, t − 1m]`, `(t − 3m, t − 2m]`, and the merge over them covers `(t − 3m, t]`. +- Counter states (`Rate`, `Increase`) merge only when their intervals on the timestamp column are disjoint (`MergeInputRule::DisjointInTime`); other families need any column disjoint. - An instant `TimeRange` (latest sample per series) does not pick rows by time, so it stays in `definition`. The walk also stops there, so for now the label matchers of an instant selector stay in `definition` too; lifting them is planned together with `rate` (Rule 1). -- The IR cannot yet write a timestamp constant, so absolute SQL time filters stay in `definition` for now. -- Absolute and relative time are never compared: a state over `(−1m, 0]` and one over `ts ∈ [t0, t1)` are treated as possibly overlapping. +- Limits of different variants never compare: a state over `ts ∈ (t − 1m, t]` and one over `ts ∈ (t0, t1]` are treated as possibly overlapping, and a relative interval and an absolute timestamp condition on one path do not combine (time then stays in `definition`). **More examples** of what goes where. Each query's quantile is answered from a KLL `SummaryAgg`: @@ -396,7 +457,7 @@ Any other shape stays in `definition`: |---|---|---|---| | SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `region ∈ {us}` | | SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `latency ∈ (−∞, 100)` | -| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | the last 3 to 2 minutes before evaluation, `(−3m, −2m]` | +| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `ts ∈ (t − 3m, t − 2m]` | | PromQL `quantile(0.99, rate(m{job="api"}[5m]))` | `rate(TimeRange(5m, Scan m {job = 'api'}))` | all of it | nothing: `job = 'api'` is under `rate`, which the walk cannot pass yet (Rule 1, later) | | PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | `rate(TimeRange(5m, Scan m))` | `value ∈ (0, ∞)`: the `Filter` is above `rate` | | PromQL `quantile(0.99, rate(a[5m]) > rate(b[5m]))` | `BinaryOp(>, rate(TimeRange(5m, Scan a)), rate(TimeRange(5m, Scan b)))` | all of it | nothing: a vector/vector comparison is a `BinaryOp`, not a `Filter` | @@ -405,7 +466,7 @@ Any other shape stays in `definition`: #### 4.2.3 Cost of deriving coverage -**When it runs.** `coverage()` derives a node's coverage the first time it is called and caches it on the node, so each node pays once. Building a `SummaryMerge` (`OperatorNode::new`) derives its coverage to reject an invalid merge and caches the result, so `coverage()` and `validate_structure` do not derive it again. `validate_structure` derives it only for a merge that was not built through `new`. +**When it runs.** `find_or_add_group` derives an aggregate group's coverage once, when it creates the group, and stores it in `GroupProperties::aggregate_coverage`; `add_candidate` derives a candidate's the same way. A `SummaryMerge` reads its inputs' stored coverage, so each state is derived once. Operator-node trees carry no coverage. **Sizes used below.** @@ -688,7 +749,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ State: series labels, state Sketch(KLL k=200) │ [[ SummaryMerge ]] - ▲ selection: time (−2m, 0] + ▲ selection: ts ∈ (t − 2m, t] │ ┌─────────────────┴─────────────────┐ │ State: series labels, │ State: series labels, @@ -696,7 +757,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ │ [[ SummaryAgg ]] tumbling window 0 [[ SummaryAgg ]] tumbling window 1 KLL k=200, per series KLL k=200, per series - selection: time (−1m, 0] selection: time (−2m, −1m] + selection: ts ∈ (t − 1m, t] selection: ts ∈ (t − 2m, t − 1m] ▲ ▲ │ │ [ TimeRange 1m ] [ TimeRange 1m ] @@ -798,26 +859,26 @@ input groups output groups | State | Query | Selection | |---|---|---| -| `A` | `quantile_over_time(0.99, m[1m])` | time `(−1m, 0]` | -| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | time `(−2m, −1m]` | -| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | time `(−90s, −30s]` | -| `D` | `quantile_over_time(0.99, n[1m])` | time `(−1m, 0]`, but over metric `n`: definition `KLL(value) per series over Scan n` | +| `A` | `quantile_over_time(0.99, m[1m])` | ts ∈ `(t − 1m, t]` | +| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | ts ∈ `(t − 2m, t − 1m]` | +| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | ts ∈ `(t − 90s, t − 30s]` | +| `D` | `quantile_over_time(0.99, n[1m])` | ts ∈ `(t − 1m, t]`, but over metric `n`: definition `KLL(value) per series over Scan n` | | Merge | Allowed? | Why | Result's selection | |---|---|---|---| -| `A + B` | ✓ | same definition, no overlap | `(−2m, 0]` (touching time windows join) | -| `A + C` | ✗ | `(−60s, −30s]` is in both, so those rows would be counted twice | | +| `A + B` | ✓ | same definition, no overlap | `(t − 2m, t]` (touching time windows join) | +| `A + C` | ✗ | `(t − 60s, t − 30s]` is in both, so those rows would be counted twice | | | `A + A` | ✗ | every row is in both | | | `A + D` | ✗ | different definitions: `D` reads metric `n`, not `m` | | | `A` + a KLL with `k = 400` | ✗ | different schema: `k` is part of the state type, so the merge is rejected before coverage is checked | | -| `(A + B)` + the state of `m[1m] offset 2m`, time `(−3m, −2m]` | ✓ | a merge has coverage like any state, so merges nest | `(−3m, 0]` | +| `(A + B)` + the state of `m[1m] offset 2m`, ts ∈ `(t − 3m, t − 2m]` | ✓ | a merge has coverage like any state, so merges nest | `(t − 3m, t]` | **Whether inputs may overlap** depends on the summary type (family and algorithm, §3). This is #592 (open), which adds `FieldDataType::family_merges` and `merge_relation`; #646 alone requires every merge to be overlap-free: | Rule | Summary types | Example | |---|---|---| -| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `(−60s, −30s]` would be counted twice | -| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `(−1m, 0]` and `(−90s, −30s]`, which together cover `(−90s, 0]` | +| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `(t − 60s, t − 30s]` would be counted twice | +| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `(t − 1m, t]` and `(t − 90s, t − 30s]`, which together cover `(t − 90s, t]` | | **cannot merge (yet)** | `CmsWithHeap`, `CountSketchWithHeap`, Theta, KMV, exact `Rate`/`IRate`/`Increase`, samples, wavelets, models | no sound merge is modeled yet, so #592 rejects them | | **right inside left** | subtraction | 5.7 | @@ -825,7 +886,7 @@ input groups output groups |---|---| | What comes out? | the children's schema; with `group_by`, its columns in order, then the state column. A `group_by` of no columns over a SQL relation's states stands for a global SQL aggregate, which has one row over no input rows: it outputs one state, empty when there is no input state | | When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra) | -| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(−2m, −1m]` and `(−1m, 0]` give `(−2m, 0]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | +| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(t − 2m, t − 1m]` and `(t − 1m, t]` give `(t − 2m, t]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | | State or value? | state in, state out | **Guarantee after a rollup.** The groups of one build partition its rows, so the states a rollup merges into one output group are disjoint, and the family-capability table's merged error applies. A rolled-up state of these families keeps the guarantee one build of the same size would have over the coarse group's rows; most are exactly that build's state: @@ -847,7 +908,7 @@ Rejected: counter states (`Rate`, `Increase`), which merge consecutive windows o **Other uses of coverage.** The planner also uses coverage to read or reuse a state without merging: - **Slice:** from a `by[region, job]` state, a query for `region = 'us'` by job reads only the `us` groups ✓. A query for `value < 50` ✗: `value` is not a grouping column, and a sketch cannot be filtered after it is built. -- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `(−5m, 0]` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. +- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `(t − 5m, t]` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. ### 5.7 Reserved operators (not implemented) @@ -888,9 +949,10 @@ OperatorNode §6.1 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output │ └── Sketch(SketchAlgorithm) `state` field has type `summary_type`) │ (θ, `SketchParams`, is on the candidate) §6.4 -├── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) -└── coverage() → SummaryCoverage §6.6 - └── definition: OperatorNode (again a node, so the same structure) +└── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) + +MemoGroup.group_properties.aggregate_coverage: SummaryCoverage §6.6 + └── definition: SummaryStateDefinition { input_group, unlifted_operators } ``` | Subsection | Defines | Used by | @@ -900,7 +962,7 @@ OperatorNode §6.1 | §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.summary_type` | | §6.4 Summary algorithms and parameters | `SketchAlgorithm`, `SketchParams`, `HydraParams`, exact/sample/wavelet/model kinds and params | the payload of each non-plain `FieldDataType` variant; θ on a candidate's `LogicalProperties` | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | -| §6.6 Summary coverage | `SummaryCoverage`, `SelectionBox`, `Constraint` | `OperatorNode::coverage()` on summary nodes | +| §6.6 Summary coverage | `SummaryCoverage`, `SummaryStateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of summary-state memo groups | **Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). @@ -1313,16 +1375,16 @@ A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarant ### 6.6 Summary coverage (`crates/types/src/ir/properties/summary_coverage.rs`) -The code for §4, as of #646. On `main` today `SummaryCoverage` is still the older declared form (`source` plus `regions`). +The code for §4. `SummaryStateDefinition` is shown in §4.2.2. ```rust pub struct SummaryCoverage { - /// What the state computes: the `SummaryAgg` and its sub-DAG, with the - /// conditions that went into `selection` taken out (§4.2.2). - pub definition: Rc, + /// What the state computes: its input group and the operators the walk + /// did not lift (§4.2.2). + pub(crate) definition: SummaryStateDefinition, /// Which rows went in: a union of boxes. A row is in the state if it is /// in at least one box. - pub selection: Vec, + pub(crate) selection: Vec, } /// One box: a row is in it when it meets every constraint. @@ -1376,9 +1438,10 @@ A PromQL range `m[r]` evaluated at t reads the samples in (t − r, t]: left-ope ```rust impl SummaryCoverage { - /// Computes the coverage of a summary node from its sub-DAG (§4.2.2). - /// Errors: `NotSummary` for a node that is not a `SummaryAgg`, - /// `SummaryTumblingWindowChain` or `SummaryMerge`; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` + /// The coverage of a summary-state group computed by `operator` over its + /// input groups (§4.2.2). Errors: `NotSummary` for an operator that is + /// not a `SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge` or a + /// `Filter` keeping whole states; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` /// that does not expand to one build over its whole window; for a /// merge, an input's own error, `EmptyMerge` (no inputs), /// `DefinitionMismatch` (inputs compute different things) or @@ -1388,9 +1451,12 @@ impl SummaryCoverage { /// not a group key of the inputs' build), `FamilyCannotRegroup` (counter /// states, a family without a merge, or a Hydra) or `RegroupedBuild` /// (the coarse build it stands for does not derive). - pub fn derive(node: &OperatorNode) -> Result; + pub(crate) fn derive_from_inputs( + operator: &Operator>, + ) -> Result; + pub(crate) fn definition_id(&self) -> String; } ``` -- `OperatorNode::new` and `validate_structure` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap; #592 relaxes the overlap check per family). The derived coverage is cached on the node. +- `find_or_add_group` and `add_candidate` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap, per the family's `MergeInputRule`). Operator-node trees have no coverage and no merge check. - A `SummaryAgg` always has coverage: what cannot go into `selection` stays in `definition`. From 77fc121d3efcd5899e002cfa6b039eee97f50c72 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:27:58 +0000 Subject: [PATCH 02/12] fix(types, stage0): one aggregate layout for schema and signs; memo lookups before derivation (#1207 PR 2 review) 1. Aggregate layout: aggregate_output_schema and the new aggregate_output_columns derive the schema and, per output column, its OutputColumnSource (InputColumn, Measure, AddedLabel) in one pass; the column signs read the sources, and a summary state's layout is summary_agg_columns, beside summary_agg_schema. Fixes a `without` count_values aggregate getting a sign for a label column its schema does not have (count_values_without_has_one_sign_per_output_column). 2. find_or_add_group looks the expression up before deriving its properties; add_candidate checks for a duplicate first and then only the node rules (check_candidate_expression), not the group properties. 3. A chain's coverage is of_summary_agg over the chain itself. 4. combined_with: one constraint intersection for the lifted columns. 5. of_window_selection walks to the chain with selected_from. 6. SourceColumn deleted: the memo's declared signs are keyed by source, then column, declared by Stage 0 as its walk reaches each source's Scan (Memo::declare_column_signs replaces Memo::new). 7. tests/memo_fixtures.rs: scan, filter, range, shift and state shared by the two memo test files; group_properties_come_from_input_groups moves to memo_group_properties.rs without its unshareable-LIMIT assertion. a. A group's serialized operator is computed once and stored with its hash bucket entry. b. depends_on_evaluation_time checks the input groups once. c. The SQL timestamp cast target is dtype_to_arrow(&DataType::Timestamp). --- crates/frontend-sql/src/sql/expr.rs | 9 +- crates/stage0-memo/src/lib.rs | 64 ++--- crates/types/src/ir/operator/arguments.rs | 10 - crates/types/src/ir/operator/asap.rs | 14 + crates/types/src/ir/operator/mod.rs | 6 +- .../src/ir/properties/summary_coverage.rs | 103 ++++---- .../types/src/ir/schema/aggregate_schema.rs | 108 ++++++-- crates/types/src/ir/schema/mod.rs | 2 +- .../src/stages/logical/column_properties.rs | 152 ++++++----- crates/types/src/stages/logical/group.rs | 63 +++-- .../src/stages/logical/memo/hash_consing.rs | 33 ++- crates/types/src/stages/logical/memo/mod.rs | 89 ++++--- crates/types/tests/main.rs | 1 + crates/types/tests/memo_fixtures.rs | 71 +++++ crates/types/tests/memo_group_properties.rs | 242 ++++++++++-------- crates/types/tests/memo_summary_coverage.rs | 146 ++--------- 16 files changed, 604 insertions(+), 509 deletions(-) create mode 100644 crates/types/tests/memo_fixtures.rs diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs index 4e51a9dc9..56043ec36 100644 --- a/crates/frontend-sql/src/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -1,16 +1,17 @@ use std::rc::Rc; -use datafusion::arrow::datatypes::{DataType as ArrowDataType, TimeUnit}; +use datafusion::arrow::datatypes::DataType as ArrowDataType; use datafusion::logical_expr::utils::{split_binary, split_conjunction}; use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; use asap_frontend_common::UnresolvedScalar as Unresolved; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::DataType; use asap_types::ir::ExprSemantics; use crate::error::SqlError as LoweringError; -use super::types::{cast_target, scalar_value_to_asap}; +use super::types::{cast_target, dtype_to_arrow, scalar_value_to_asap}; use super::SqlLowerer; impl SqlLowerer<'_> { @@ -153,9 +154,7 @@ impl SqlLowerer<'_> { unreachable!() }; let to = match c.field.data_type() { - ArrowDataType::Timestamp(..) => { - ArrowDataType::Timestamp(TimeUnit::Millisecond, None) - } + ArrowDataType::Timestamp(..) => dtype_to_arrow(&DataType::Timestamp), other => other.clone(), }; match sv.cast_to(&to) { diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index 9ba31b4b1..2e120c3dc 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -8,11 +8,10 @@ //! where the one expression equality is; no other stage crate depends on //! it, and the planner facade calls it first. -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use std::rc::Rc; use asap_types::accuracy::AccuracyTarget; -use asap_types::ir::operator::SourceColumn; use asap_types::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, SchemaDerivationError}; use asap_types::stages::{Memo, MemoGroupId, MemoQuery, ValueSign}; use asap_types::workload::bounds::{series_read, source_name, values_per_result}; @@ -23,7 +22,8 @@ use asap_types::workload::{DataWorkload, PrometheusMetricType}; /// none), every group without candidates. Each query is added with its /// $n_q$, the number of values one evaluation returns, which `data` bounds /// ([`values_per_result`]); a `Scan`'s column signs are the ones `data` -/// declares ([`declared_column_signs`]). Fails with the first node that +/// declares ([`declared_column_signs`]), declared to the memo as the walk +/// reaches the source's first `Scan`. Fails with the first node that /// breaks the node rules over its children's groups /// ([`Memo::find_or_add_group`]); the facade validates the roots first, so /// that is a frontend bug. @@ -37,11 +37,11 @@ pub fn memo_of_queries( accuracy_targets.len(), "one accuracy target per query" ); - let mut memo = Memo::new(declared_column_signs(roots, data)); + let mut memo = Memo::default(); let mut group_of_node = GroupOfNode::new(); for root in roots { for node in root.operator_refs() { - group_of(&mut memo, &mut group_of_node, node)?; + group_of(&mut memo, &mut group_of_node, node, data)?; } } let group = |node: &Rc| group_of_node[&Rc::as_ptr(node)]; @@ -64,42 +64,19 @@ pub fn memo_of_queries( Ok(memo) } -/// The sign `data` declares for each column of each source the queries -/// scan ([`SourceFacts::column_signs`](asap_types::workload::SourceFacts::column_signs), +/// The sign `data` declares for each column of `source`, by column name +/// ([`SourceFacts::column_signs`](asap_types::workload::SourceFacts::column_signs), /// a histogram's or summary's for its series, [`DataWorkload::series_facts`]); /// a declared counter's `value` is non-negative without a declaration. -fn declared_column_signs( - roots: &[QueryRoot], - data: &DataWorkload, -) -> BTreeMap { - let mut signs = BTreeMap::new(); - let mut visited = HashSet::new(); - let mut pending: Vec<&Rc> = - roots.iter().flat_map(QueryRoot::operator_refs).collect(); - while let Some(node) = pending.pop() { - if !visited.insert(Rc::as_ptr(node)) { - continue; - } - pending.extend(node.children()); - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.operator else { - continue; - }; - let source = source_name(source); - let Some((facts, series_type)) = data.series_facts(source) else { - continue; - }; - let column = |name: &str| SourceColumn { - source: source.to_owned(), - column: name.to_owned(), - }; - for (name, sign) in &facts.column_signs { - signs.insert(column(name), *sign); - } - if series_type == Some(PrometheusMetricType::Counter) { - signs - .entry(column("value")) - .or_insert(ValueSign::NonNegative); - } +fn declared_column_signs(source: &str, data: &DataWorkload) -> BTreeMap { + let Some((facts, series_type)) = data.series_facts(source) else { + return BTreeMap::new(); + }; + let mut signs = facts.column_signs.clone(); + if series_type == Some(PrometheusMetricType::Counter) { + signs + .entry("value".to_owned()) + .or_insert(ValueSign::NonNegative); } signs } @@ -111,16 +88,23 @@ type GroupOfNode = HashMap<*const OperatorNode, MemoGroupId>; /// The group of `node`: its operator over its children's groups (each /// found or added first), found or added by [`Memo::find_or_add_group`]. +/// A `Scan`'s source has its column signs declared to `memo` first +/// ([`declared_column_signs`]). fn group_of( memo: &mut Memo, group_of_node: &mut GroupOfNode, node: &Rc, + data: &DataWorkload, ) -> Result { if let Some(&group) = group_of_node.get(&Rc::as_ptr(node)) { return Ok(group); } for child in node.children() { - group_of(memo, group_of_node, child)?; + group_of(memo, group_of_node, child, data)?; + } + if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.operator { + let source = source_name(source); + memo.declare_column_signs(source, declared_column_signs(source, data)); } let expression = node .operator diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index 603f4a4aa..e2a6ef1da 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -178,16 +178,6 @@ pub enum Source { Table { table_ref: String }, } -/// A column of a source (a `Scan`'s metric or table), by name: what the data -/// workload declares facts about, such as a column's sign -/// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SourceColumn { - pub source: String, - pub column: String, -} - /// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's /// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison /// operator has exactly one representation (and one `Display`) across the IR. diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index fc7f44900..e63fe3f7a 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -760,6 +760,20 @@ pub(crate) fn summary_agg_schema( Ok(schema) } +/// Where each column of [`summary_agg_schema`]'s output comes from: the +/// aggregate layout of `reduction` with the state in the one measure's place +/// (`OutputColumnSource::Measure(0)`). +pub(crate) fn summary_agg_columns( + child: &Schema, + reduction: &Reduction, +) -> Result, SchemaDerivationError> { + crate::ir::schema::aggregate_output_columns( + child, + reduction, + &[crate::ir::operator::AggIntent::Sum { col: None }], + ) +} + /// The schema of a `SummaryTumblingWindowChain` of `summary_type` by `reduction` /// over rows of schema `child`: a state with a time index, one row per /// window and key, (window_end, keys…, state). `window_end` (column 0, the diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index c8e36b8e4..b9b53b2a4 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -28,7 +28,7 @@ pub use agg_intent::{ }; pub use arguments::{ AtModifier, BinaryOpKind, ColState, GroupKeys, GroupSide, InfoMatcher, JoinKind, - PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, Source, SourceColumn, - TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, - WindowFrameOffset, WindowFrameUnits, WindowFuncKind, + PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, Source, TimeShift, + VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, WindowFrameOffset, + WindowFrameUnits, WindowFuncKind, }; diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 651626c34..f1734c116 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -317,27 +317,9 @@ impl SummaryCoverage { Some(build) => of_summary_agg(&build), None => Err(CoverageError::NotSummary), }, - Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), - Operator::ASAP(op @ ASAPOp::SummaryTumblingWindowChain { .. }) => { - let ASAPOp::SummaryTumblingWindowChain { - child, - summary_type, - input, - reduction, - filter, - .. - } = as_built(op) - else { - unreachable!("matched a chain"); - }; - of_summary_agg(&ASAPOp::SummaryAgg { - child, - summary_type, - input, - reduction, - filter, - }) - } + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => of_summary_agg(&as_built(op)), Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { @@ -591,18 +573,27 @@ fn time_column_of_build_input(state: &I) -> Option>) -> Result { - let ASAPOp::SummaryAgg { + let (ASAPOp::SummaryAgg { + child, + summary_type: family, + input, + reduction, + filter, + } + | ASAPOp::SummaryTumblingWindowChain { child, summary_type: family, input, reduction, filter, - } = build + .. + }) = build else { - unreachable!("called on a SummaryAgg"); + unreachable!("called on a SummaryAgg or a chain"); }; of_aggregate(child, filter.as_ref(), |residual_filter| { Operator::ASAP(ASAPOp::SummaryAgg { @@ -699,11 +690,7 @@ fn of_aggregate( let column = chain[depth].children()[0].schema().time_index?; let column = column_at_top(column, &chain[..=depth])?; column_identity(top, column)?; - let combined = match lifted.get(&column) { - Some(existing) => existing.intersect(&constraint)?, - None => constraint, - }; - Some((column, combined)) + Some((column, combined_with(lifted.get(&column), constraint)?)) }); let lift_time = time.is_some(); lifted.extend(time); @@ -763,11 +750,7 @@ fn residual( let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; column_identity(top, column)?; - let combined = match lifted.get(&column) { - Some(existing) => existing.intersect(&constraint)?, - None => constraint, - }; - Some((column, combined)) + Some((column, combined_with(lifted.get(&column), constraint)?)) }); match lift { Some((column, constraint)) => { @@ -783,6 +766,15 @@ fn residual( } } +/// `constraint` combined with what is lifted on its column already +/// (`existing`); `None` when the two do not combine. +fn combined_with(existing: Option<&Constraint>, constraint: Constraint) -> Option { + match existing { + Some(existing) => existing.intersect(&constraint), + None => Some(constraint), + } +} + /// The memo group `input` is; `None` for a rule's temporary build. fn group_id(input: &GroupInput<'_>) -> Option { match input { @@ -894,17 +886,23 @@ fn of_window_selection( ) -> Result { let mut links = vec![selection.clone()]; let mut below = *selection.children()[0]; - loop { - match below.operator() { - link @ Operator::NonASAP( - NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, - ) => { - below = child; - links.push(link); - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => break, - _ => return Err(CoverageError::NotSummary), - } + while let Some(child) = selected_from(&below) { + links.push(below.operator()); + below = child; + } + let only_time_selections = links.iter().all(|link| { + matches!( + link, + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + ) + }); + if !only_time_selections + || !matches!( + below.operator(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + { + return Err(CoverageError::NotSummary); } let chain = below .aggregate_coverage() @@ -916,14 +914,9 @@ fn of_window_selection( .iter() .map(|selection_box| { let mut columns = selection_box.columns.clone(); - let combined = match columns.get(&time_column) { - Some(existing) => existing.intersect(&constraint), - None => Some(constraint.clone()), - }; - columns.insert( - time_column.clone(), - combined.ok_or(CoverageError::NotSummary)?, - ); + let combined = combined_with(columns.get(&time_column), constraint.clone()) + .ok_or(CoverageError::NotSummary)?; + columns.insert(time_column.clone(), combined); Ok(SelectionBox { columns }) }) .collect::>()?; diff --git a/crates/types/src/ir/schema/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs index 2c1113534..cf8e920d4 100644 --- a/crates/types/src/ir/schema/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -10,6 +10,18 @@ use crate::ir::operator::AggIntent; use crate::ir::scalar::ColumnRef; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; + +/// Where one output column of an aggregate comes from: an input column it +/// keeps, a measure's value, or a label column the measure adds +/// (`count_values`' and `absent`'s). `sum by (job) (x)` → +/// `[InputColumn(job), Measure(0)]`. The one layout the output schema +/// ([`aggregate_output_schema`]) and the group properties read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutputColumnSource { + InputColumn(ColumnId), + Measure(usize), + AddedLabel, +} /// Output schema of a *per-series* window/range reduction (`rate`/`increase`, /// or an `*_over_time` reducer under a time `Window`). Such a reduction emits /// one value per series, so every label column of `input` is preserved and only @@ -18,7 +30,7 @@ use crate::ir::SchemaDerivationError; fn per_series_reduction_schema( input: &Schema, agg: &AggIntent, -) -> Result { +) -> Result<(Schema, Vec), SchemaDerivationError> { let vi = if let Some(index) = agg.input_cols().first() { *index } else { @@ -34,6 +46,10 @@ fn per_series_reduction_schema( ))); } let mut columns = input.fields.clone(); + let mut sources: Vec = (0..columns.len()) + .map(OutputColumnSource::InputColumn) + .collect(); + sources[vi] = OutputColumnSource::Measure(0); { let mut out = agg.output_column(&columns[vi]); out.name = "value".into(); @@ -44,14 +60,17 @@ fn per_series_reduction_schema( out.dtype = FieldDataType::Plain(DataType::Float64); columns[vi] = out; } - Ok(Schema { - fields: columns, - time_index: input.time_index, - unique_keys: input.unique_keys.clone(), - // Per-series reduction is label-preserving: it inherits its input's - // completeness (an open scan stays open; a closed one stays closed). - closed: input.closed, - }) + Ok(( + Schema { + fields: columns, + time_index: input.time_index, + unique_keys: input.unique_keys.clone(), + // Per-series reduction is label-preserving: it inherits its input's + // completeness (an open scan stays open; a closed one stays closed). + closed: input.closed, + }, + sources, + )) } /// The output schema of an `Aggregate { reduction, measures }` over `in_schema` — @@ -71,6 +90,26 @@ pub fn aggregate_output_schema( measures: &[AggIntent], output_names: &[String], ) -> Result { + Ok(aggregate_output(in_schema, reduction, measures, output_names)?.0) +} + +/// Where each column of [`aggregate_output_schema`]'s output comes from, in +/// its order. +pub fn aggregate_output_columns( + in_schema: &Schema, + reduction: &Reduction, + measures: &[AggIntent], +) -> Result, SchemaDerivationError> { + Ok(aggregate_output(in_schema, reduction, measures, &[])?.1) +} + +/// [`aggregate_output_schema`] and [`aggregate_output_columns`] in one pass. +fn aggregate_output( + in_schema: &Schema, + reduction: &Reduction, + measures: &[AggIntent], + output_names: &[String], +) -> Result<(Schema, Vec), SchemaDerivationError> { let by = match reduction { Reduction::PerEntity => { debug_assert_eq!( @@ -93,6 +132,7 @@ pub fn aggregate_output_schema( } let mut out_cols: Vec = Vec::with_capacity(by.len() + measures.len()); + let mut sources: Vec = Vec::with_capacity(by.len() + measures.len()); for &id in by.keys() { let c = in_schema .fields @@ -102,6 +142,7 @@ pub fn aggregate_output_schema( in_schema.fields.len(), ))?; out_cols.push(c.clone()); + sources.push(OutputColumnSource::InputColumn(id)); } let value_col_idx = crate::ir::scalar::column_resolution::resolve_column_ref( &ColumnRef::SampleValue, @@ -126,12 +167,14 @@ pub fn aggregate_output_schema( if let AggIntent::CountValues { label } = intent { if !out_cols.iter().any(|c| c.name == *label) { out_cols.push(Field::plain(label.clone(), DataType::Utf8, false)); + sources.push(OutputColumnSource::AddedLabel); } let mut cnt = intent.output_column(&probe); if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { cnt.name = name.clone(); } out_cols.push(cnt); + sources.push(OutputColumnSource::Measure(i)); continue; } // `absent(v)` emits its synthesized labels as `Utf8` columns before @@ -142,6 +185,7 @@ pub fn aggregate_output_schema( .keys() .map(|name| Field::plain(name.clone(), DataType::Utf8, false)), ); + sources.extend(labels.keys().map(|_| OutputColumnSource::AddedLabel)); } // Only the output *type* is read from here, so the leading column is // enough for the multi-column intents: `Cardinality` and `PearsonCorr` @@ -168,6 +212,7 @@ pub fn aggregate_output_schema( out.name = name.clone(); } out_cols.push(out); + sources.push(OutputColumnSource::Measure(i)); } // `count_values` groups by (by-keys ∪ the synthesized value label), so the // by-keys alone are not a unique key — be conservative and claim none. @@ -179,15 +224,18 @@ pub fn aggregate_output_schema( } else { vec![(0..by.len()).collect()] }; - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys, - // A cross-series aggregate enumerates exactly `by ++ measures`, so its output - // is closed even over an open input — this is where an open schema - // freezes to closed. - closed: true, - }) + Ok(( + Schema { + fields: out_cols, + time_index: None, + unique_keys, + // A cross-series aggregate enumerates exactly `by ++ measures`, so its output + // is closed even over an open input — this is where an open schema + // freezes to closed. + closed: true, + }, + sources, + )) } /// Output schema of a `without(excluded)` aggregate: the kept labels (every @@ -201,7 +249,7 @@ fn without_output_schema( excluded: &[ColumnId], measures: &[AggIntent], output_names: &[String], -) -> Result { +) -> Result<(Schema, Vec), SchemaDerivationError> { for &id in excluded { if id >= in_schema.fields.len() { return Err(SchemaDerivationError::InvalidGroupByColumn( @@ -211,6 +259,7 @@ fn without_output_schema( } } let mut out_cols: Vec = Vec::new(); + let mut sources: Vec = Vec::new(); for (i, col) in in_schema.fields.iter().enumerate() { let is_time = in_schema.time_index == Some(i); let is_value = crate::ir::scalar::column_resolution::resolve_column_ref( @@ -221,6 +270,7 @@ fn without_output_schema( == Some(i); if !is_time && !is_value && !excluded.contains(&i) { out_cols.push(col.clone()); + sources.push(OutputColumnSource::InputColumn(i)); } } let probe = in_schema @@ -249,15 +299,19 @@ fn without_output_schema( out.name = name.clone(); } out_cols.push(out); + sources.push(OutputColumnSource::Measure(i)); } - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys: Vec::new(), - // The kept label set is runtime-only, so — unlike `by` — this does not - // freeze the open schema to closed. - closed: false, - }) + Ok(( + Schema { + fields: out_cols, + time_index: None, + unique_keys: Vec::new(), + // The kept label set is runtime-only, so — unlike `by` — this does not + // freeze the open schema to closed. + closed: false, + }, + sources, + )) } #[cfg(test)] diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 72e3e5627..8d5981154 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -18,7 +18,7 @@ pub mod error; pub mod series_identity; pub mod topk; -pub use aggregate_schema::aggregate_output_schema; +pub use aggregate_schema::{aggregate_output_columns, aggregate_output_schema, OutputColumnSource}; use serde::{Deserialize, Serialize}; diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index b2fe79c57..fe096748a 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -6,15 +6,13 @@ use std::collections::BTreeMap; use super::{MemoGroupAsInput, ValueSign}; -use crate::ir::operator::arguments::{ - BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction, SourceColumn, -}; -use crate::ir::operator::asap::{summary_agg_schema, summary_build}; +use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction}; +use crate::ir::operator::asap::{summary_agg_columns, summary_agg_schema, summary_build}; use crate::ir::operator::input::OperatorInput; use crate::ir::operator::AggIntent; use crate::ir::scalar::expr_ir::ScalarValue; use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; -use crate::ir::schema::{FieldDataType, Schema}; +use crate::ir::schema::{aggregate_output_columns, FieldDataType, OutputColumnSource, Schema}; use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; use crate::workload::bounds::source_name; @@ -28,7 +26,7 @@ type Input<'m> = MemoGroupAsInput<'m>; pub(crate) fn output_column_sign( operator: &Operator>, output: &Schema, - declared: &BTreeMap, + declared: &BTreeMap>, ) -> Vec { let signs = match operator { Operator::NonASAP(op) => non_asap_signs(op, output, declared), @@ -50,22 +48,24 @@ fn signs_of(input: &Input<'_>) -> Vec { fn non_asap_signs( op: &NonASAPOp>, output: &Schema, - declared: &BTreeMap, + declared: &BTreeMap>, ) -> Vec { use NonASAPOp::*; let unknown = || vec![Unknown; output.fields.len()]; match op { - Scan { source, schema, .. } => schema - .fields - .iter() - .map(|field| { - let column = SourceColumn { - source: source_name(source).to_owned(), - column: field.name.clone(), - }; - declared.get(&column).copied().unwrap_or(Unknown) - }) - .collect(), + Scan { source, schema, .. } => { + let declared = declared.get(source_name(source)); + schema + .fields + .iter() + .map(|field| { + declared + .and_then(|columns| columns.get(&field.name)) + .copied() + .unwrap_or(Unknown) + }) + .collect() + } Values { .. } | PromqlVectorFromScalar(_) => unknown(), Filter { child, .. } | Sort { child, .. } @@ -264,65 +264,65 @@ fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { } } -/// An aggregate's, in `aggregate_output_schema`'s layout: a per-entity -/// reduction keeps its input's columns in place, its value replaced by the -/// measure's; a `by` reduction lists its keys, then its measures; a -/// `without` reduction the input's other label columns, then its measures. +/// An aggregate's, by where each output column comes from +/// ([`aggregate_output_columns`]): a kept input column keeps its sign, a +/// measure's value has [`measure_sign`] of its input value's (its first +/// input column, else the column `aggregate_output_schema` types it by), and +/// a label a measure adds is `Unknown`. fn aggregate_signs( child: &Input<'_>, reduction: &Reduction, measures: &[AggIntent], output: &Schema, +) -> Vec { + let Ok(sources) = aggregate_output_columns(child.schema(), reduction, measures) else { + return vec![Unknown; output.fields.len()]; + }; + let input = signs_of(child); + signs_from_sources(child, &sources, |index, value| { + let measure = &measures[index]; + let value = measure.input_cols().first().copied().or(value); + measure_sign( + measure, + value + .and_then(|value| input.get(value).copied()) + .unwrap_or(Unknown), + ) + }) +} + +/// The sign of each output column from `sources`: an input column keeps +/// its sign, measure `index` has `measure(index, value)`, where `value` is +/// the column `aggregate_output_schema` types a measure without an input +/// column by (the sample value, else the first column not kept), and an +/// added label is `Unknown`. +fn signs_from_sources( + child: &Input<'_>, + sources: &[OutputColumnSource], + measure: impl Fn(usize, Option) -> ValueSign, ) -> Vec { let input = signs_of(child); let schema = child.schema(); - let sample_value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); - let value_of = |measure: &AggIntent| measure.input_cols().first().copied().or(sample_value); - let keys: Vec = match reduction { - Reduction::PerEntity => { - let mut signs = input.clone(); - if let Some(measure) = measures.first() { - if let Some(value) = value_of(measure).filter(|&value| value < signs.len()) { - signs[value] = measure_sign(measure, input[value]); - } - } - return signs; - } - Reduction::Reduce(by) if by.is_without() => (0..schema.fields.len()) - .filter(|&column| { - schema.time_index != Some(column) - && sample_value != Some(column) - && !by.keys().contains(&column) - }) - .map(|column| input[column]) - .collect(), - Reduction::Reduce(by) => by - .keys() - .iter() - .map(|&key| input.get(key).copied().unwrap_or(Unknown)) - .collect(), - }; - let measured: Vec = measures + let kept: Vec = sources .iter() - .flat_map(|measure| { - let sign = measure_sign( - measure, - value_of(measure) - .and_then(|value| input.get(value).copied()) - .unwrap_or(Unknown), - ); - // `count_values` and `absent` add label columns before their value. - let labels = match measure { - AggIntent::CountValues { label } => { - usize::from(!output.fields[..keys.len()].iter().any(|f| f.name == *label)) - } - AggIntent::Absent { labels } => labels.len(), - _ => 0, - }; - std::iter::repeat_n(Unknown, labels).chain([sign]) + .filter_map(|source| match source { + OutputColumnSource::InputColumn(column) => Some(*column), + _ => None, }) .collect(); - keys.into_iter().chain(measured).collect() + let value = resolve_column_ref(&ColumnRef::SampleValue, schema) + .ok() + .or_else(|| (0..schema.fields.len()).find(|column| !kept.contains(column))); + sources + .iter() + .map(|source| match source { + OutputColumnSource::InputColumn(column) => { + input.get(*column).copied().unwrap_or(Unknown) + } + OutputColumnSource::Measure(index) => measure(*index, value), + OutputColumnSource::AddedLabel => Unknown, + }) + .collect() } /// The sign of `measure`'s value over an input value of sign `input`. @@ -357,21 +357,18 @@ fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { } } -/// A summary state's, in `state_schema`'s layout (an aggregate of one -/// measure): its key columns keep their input's sign and its state column -/// records the sign of the values it is updated with. +/// A summary state's, in its layout ([`summary_agg_columns`]): its key +/// columns keep their input's sign and its state column records the sign of +/// the values it is updated with. fn state_signs( child: &Input<'_>, update: &SummaryUpdate, reduction: &Reduction, state_schema: &Schema, ) -> Vec { - let mut signs = aggregate_signs( - child, - reduction, - &[AggIntent::Sum { col: None }], - state_schema, - ); + let Ok(sources) = summary_agg_columns(child.schema(), reduction) else { + return vec![Unknown; state_schema.fields.len()]; + }; let updated_with = match &update.weight { SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) .ok() @@ -380,10 +377,7 @@ fn state_signs( SummaryInputExpr::Constant(value) => sign_of_number(*value), SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, }; - if let Some(state) = state_schema.fields.iter().position(|f| !f.is_plain()) { - signs[state] = updated_with; - } - signs + signs_from_sources(child, &sources, |_, _| updated_with) } /// A readout of summary state: the state's plain columns keep their sign diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 30fcdab52..e83f95adc 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -95,19 +95,16 @@ pub enum ValueSign { Unknown, } -/// The properties of a group, or a candidate, computing `expression` over -/// its input groups: the rules that check operator nodes, reading each -/// input group's `group_properties` and `expression` -/// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression -/// breaks them (schema derivation, input validation, merge check). -pub(super) fn derive_group_properties( - memo: &Memo, - expression: &Operator, -) -> Result { - let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); +/// The node rules every group and candidate expression must pass, over +/// its input groups ([`MemoGroupAsInput`]), never a node tree: schema +/// derivation, input validation and, for a merge, the coverage check. +/// Returns the output schema and the aggregate coverage. +fn check_node_rules( + operator: &Operator>, +) -> Result<(Schema, Option), SchemaDerivationError> { let schema = operator.output_schema()?; operator.validate_inputs()?; - let aggregate_coverage = match SummaryCoverage::derive_from_inputs(&operator) { + let aggregate_coverage = match SummaryCoverage::derive_from_inputs(operator) { Ok(coverage) => Some(coverage), // A merge is valid only over states of one definition it may merge. Err(error) if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) => { @@ -115,6 +112,29 @@ pub(super) fn derive_group_properties( } Err(_) => None, }; + Ok((schema, aggregate_coverage)) +} + +/// The output schema and kind of a candidate computing `expression` over +/// its input groups, once it passes the node rules ([`check_node_rules`]). +pub(super) fn check_candidate_expression( + memo: &Memo, + expression: &Operator, +) -> Result<(Schema, OperatorResultKind), SchemaDerivationError> { + let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); + let (schema, _) = check_node_rules(&operator)?; + Ok((schema, operator.output_kind())) +} + +/// The properties of a group computing `expression` over its input +/// groups, once it passes the node rules ([`check_node_rules`]), reading +/// each input group's `group_properties` and `expression`. +pub(super) fn derive_group_properties( + memo: &Memo, + expression: &Operator, +) -> Result { + let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); + let (schema, aggregate_coverage) = check_node_rules(&operator)?; Ok(GroupProperties { column_sign: output_column_sign(&operator, &schema, &memo.declared_column_signs), depends_on_evaluation_time: depends_on_evaluation_time(&operator), @@ -137,8 +157,8 @@ fn depends_on_evaluation_time(operator: &Operator>) -> bool ScalarExpr::EvalTimestamp | ScalarExpr::CurrentTimestamp ) || expr.children().into_iter().any(reads_evaluation_time) } - match operator { - Operator::NonASAP(NonASAPOp::TimeRange { .. }) => true, + let reads_it = match operator { + Operator::NonASAP(NonASAPOp::TimeRange { .. }) => return true, Operator::NonASAP(NonASAPOp::TimeShift { shift: TimeShift { @@ -146,19 +166,14 @@ fn depends_on_evaluation_time(operator: &Operator>) -> bool .. }, .. - }) => false, - Operator::NonASAP(op) => { - op.scalar_exprs().into_iter().any(reads_evaluation_time) - || operator.children().into_iter().any(depends_on_it) - } + }) => return false, + Operator::NonASAP(op) => op.scalar_exprs().into_iter().any(reads_evaluation_time), Operator::ASAP( ASAPOp::SummaryAgg { filter, .. } | ASAPOp::SummaryTumblingWindowChain { filter, .. }, - ) => { - filter.iter().any(|pred| reads_evaluation_time(&pred.0)) - || operator.children().into_iter().any(depends_on_it) - } - Operator::ASAP(_) => operator.children().into_iter().any(depends_on_it), - } + ) => filter.iter().any(|pred| reads_evaluation_time(&pred.0)), + Operator::ASAP(_) => false, + }; + reads_it || operator.children().into_iter().any(depends_on_it) } fn depends_on_it(input: &MemoGroupAsInput<'_>) -> bool { diff --git a/crates/types/src/stages/logical/memo/hash_consing.rs b/crates/types/src/stages/logical/memo/hash_consing.rs index 3a9925050..a7c4f26ef 100644 --- a/crates/types/src/stages/logical/memo/hash_consing.rs +++ b/crates/types/src/stages/logical/memo/hash_consing.rs @@ -33,19 +33,38 @@ pub(super) fn same_expression(left: &Operator, right: &Operator) -> Option { + serde_json::to_string(&operator_and_arguments(expression)).ok() +} + /// A hash consistent with [`same_expression`], used only to narrow the -/// groups compared: SipHash over the canonical JSON of the operator and its -/// arguments, then the input groups. A non-finite `f64` serializes as -/// `null`, which only widens one bucket; [`same_expression`] decides. -pub(super) fn expression_hash(expression: &Operator) -> u64 { +/// groups compared: SipHash over `serialized`, the expression's +/// [`serialized_operator`], then the input groups. A non-finite `f64` +/// serializes as `null`, which only widens one bucket; +/// [`same_serialized_expression`] decides. +pub(super) fn expression_hash(expression: &Operator, serialized: Option<&str>) -> u64 { let mut hasher = DefaultHasher::new(); - serde_json::to_string(&operator_and_arguments(expression)) - .unwrap_or_default() - .hash(&mut hasher); + serialized.unwrap_or_default().hash(&mut hasher); expression.children().hash(&mut hasher); hasher.finish() } +/// [`same_expression`] with each side's [`serialized_operator`] computed +/// already. +pub(super) fn same_serialized_expression( + (left, left_serialized): (&Operator, Option<&str>), + (right, right_serialized): (&Operator, Option<&str>), +) -> bool { + left.children() == right.children() + && operator_and_arguments(left) == operator_and_arguments(right) + && left_serialized.is_some() + && left_serialized == right_serialized +} + /// Typed equality, confirmed by equal serializations: numeric `PartialEq` /// alone conflates signed zeros, and `SketchParams`' equality omits the /// recorded error and failure probability. The serialized check is diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index e9c1c56e0..e16fc00e6 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -6,9 +6,8 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; mod hash_consing; -use super::group::derive_group_properties; +use super::group::{check_candidate_expression, derive_group_properties}; use crate::accuracy::AccuracyTarget; -use crate::ir::operator::arguments::SourceColumn; use crate::ir::Operator; use crate::ir::QueryRoot; use crate::ir::SchemaDerivationError; @@ -16,7 +15,10 @@ use crate::stages::{ CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalRequirements, MemoGroup, MemoGroupId, ValueSign, }; -use hash_consing::{expression_hash, same_expression, same_expression_and_logical_properties}; +use hash_consing::{ + expression_hash, same_expression_and_logical_properties, same_serialized_expression, + serialized_operator, +}; /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with /// its candidates ([`MemoGroup::candidates`]: empty after Stage 0), and each @@ -31,10 +33,11 @@ pub struct Memo { /// The groups [`Self::find_or_add_group`] may return, by /// [`expression_hash`] of their expression: every group whose sharing /// is legal ([`GroupProperties::shareable_when_equal`]). - groups_by_expression_hash: HashMap>, + groups_by_expression_hash: HashMap)>>, /// The sign the data workload declares for each source column it - /// declares one for: a `Scan`'s [`GroupProperties::column_sign`]. - pub(super) declared_column_signs: BTreeMap, + /// declares one for, by source, then column name: a `Scan`'s + /// [`GroupProperties::column_sign`]. + pub(super) declared_column_signs: BTreeMap>, } /// One query of the memo: its root over groups, its accuracy target and @@ -52,13 +55,17 @@ pub struct MemoQuery { } impl Memo { - /// A memo without groups whose `Scan`s read `declared_column_signs` - /// (an undeclared source column's sign is `Unknown`). - pub fn new(declared_column_signs: BTreeMap) -> Self { - Self { - declared_column_signs, - ..Self::default() - } + /// Declares the sign of each column of `source` its data declares + /// (`column_signs`, by column name), which every `Scan` of `source` + /// added afterwards reads; an undeclared source column's sign is + /// `Unknown`. Stage 0 declares each source before its first `Scan`. + pub fn declare_column_signs( + &mut self, + source: &str, + column_signs: BTreeMap, + ) { + self.declared_column_signs + .insert(source.to_owned(), column_signs); } /// The groups, by id. @@ -90,22 +97,26 @@ impl Memo { &mut self, expression: Operator, ) -> Result { - let group_properties = derive_group_properties(self, &expression)?; - let hash = expression_hash(&expression); - let shareable = group_properties.shareable_when_equal; - if shareable { - if let Some(&group) = self - .groups_by_expression_hash - .get(&hash) - .and_then(|groups| { - groups.iter().find(|&&group| { - same_expression(&self.groups[group.0 as usize].expression, &expression) - }) + let serialized = serialized_operator(&expression); + let hash = expression_hash(&expression, serialized.as_deref()); + // Only shareable groups are listed by hash, and an equal expression + // is as shareable as the group it equals, which passed the node rules. + if let Some((group, _)) = self + .groups_by_expression_hash + .get(&hash) + .and_then(|groups| { + groups.iter().find(|(group, known)| { + same_serialized_expression( + (&self.groups[group.0 as usize].expression, known.as_deref()), + (&expression, serialized.as_deref()), + ) }) - { - return Ok(group); - } + }) + { + return Ok(*group); } + let group_properties = derive_group_properties(self, &expression)?; + let shareable = group_properties.shareable_when_equal; let id = MemoGroupId( u32::try_from(self.groups.len()).expect("a memo has at most u32::MAX groups"), ); @@ -123,7 +134,7 @@ impl Memo { self.groups_by_expression_hash .entry(hash) .or_default() - .push(id); + .push((id, serialized)); } Ok(id) } @@ -136,7 +147,7 @@ impl Memo { /// group and every group below it. /// /// Fails, leaving the memo unchanged, when the candidate's expression - /// breaks the node rules a group's does ([`derive_group_properties`]), + /// breaks the node rules a group's does ([`check_candidate_expression`]), /// or when its output schema or kind differs from the group's: then it /// does not compute the group's result. pub fn add_candidate( @@ -144,16 +155,8 @@ impl Memo { group: MemoGroupId, candidate: CandidateForMemoGroup, ) -> Result { - let output = derive_group_properties(self, &candidate.expression.operator) - .map_err(CandidateError::Schema)?; - let group_properties = &self.groups[group.0 as usize].group_properties; - if output.schema != group_properties.schema - || output.result_kind != group_properties.result_kind - { - return Err(CandidateError::OutputDiffersFromGroup { group }); - } - let candidates = &mut self.groups[group.0 as usize].candidates; - if let Some(index_in_group) = candidates + if let Some(index_in_group) = self.groups[group.0 as usize] + .candidates .iter() .position(|known| same_expression_and_logical_properties(known, &candidate)) { @@ -162,6 +165,14 @@ impl Memo { index_in_group, }); } + let (schema, result_kind) = + check_candidate_expression(self, &candidate.expression.operator) + .map_err(CandidateError::Schema)?; + let group_properties = &self.groups[group.0 as usize].group_properties; + if schema != group_properties.schema || result_kind != group_properties.result_kind { + return Err(CandidateError::OutputDiffersFromGroup { group }); + } + let candidates = &mut self.groups[group.0 as usize].candidates; let input_groups: Vec = candidate .expression .operator diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index e4f12481b..952ab82fd 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,7 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod memo_fixtures; mod memo_group_properties; mod memo_summary_coverage; mod physical_export; diff --git a/crates/types/tests/memo_fixtures.rs b/crates/types/tests/memo_fixtures.rs new file mode 100644 index 000000000..9ae9243d5 --- /dev/null +++ b/crates/types/tests/memo_fixtures.rs @@ -0,0 +1,71 @@ +//! The memo expressions `memo_group_properties` and `memo_summary_coverage` +//! build their examples from. +use std::time::Duration; + +use asap_types::ir::operator::arguments::{AtModifier, Reduction, Source, TimeShift}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; +use asap_types::stages::MemoGroupId; +use asap_types::summary_type::SummaryUpdate; + +/// `Scan metric{predicates}`: `[ts, value, columns…]`, `ts` its time index. +pub fn scan( + metric: &str, + columns: &[(&str, DataType)], + predicates: Vec>, +) -> Operator { + let fields = [ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ] + .into_iter() + .chain( + columns + .iter() + .map(|(name, dtype)| Field::plain(*name, dtype.clone(), false)), + ) + .collect(); + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: predicates.into_iter().map(Predicate).collect(), + schema: Schema::with_time_index(fields, 0, vec![]), + }) +} + +pub fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + }) +} + +/// `TimeRange[width_ms]`: a PromQL range `[w]`. +pub fn range(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(width_ms as u64), + kind: TimeRangeKind::Range, + child, + }) +} + +/// `TimeShift(offset_ms, at)`: PromQL `offset` and `@`. +pub fn shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { offset_ms, at }, + child, + }) +} + +/// `SummaryAgg[summary_type]` of the sample value per series. +pub fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 84930531b..229ebfff7 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -8,44 +8,35 @@ //! gauge [ts, value, host] nothing declared //! ``` use std::collections::BTreeMap; -use std::time::Duration; -use asap_types::ir::operator::arguments::{ - AtModifier, GroupKeys, JoinKind, Reduction, Source, SourceColumn, TimeShift, -}; +use asap_types::ir::operator::arguments::{AtModifier, GroupKeys, JoinKind, Reduction, Source}; use asap_types::ir::operator::AggIntent; + +use crate::memo_fixtures::{filter, range, scan, shift, state}; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use asap_types::ir::scalar::{ColumnRef, ProjectItem}; +use asap_types::ir::scalar::ProjectItem; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind, + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, }; use asap_types::stages::{Memo, MemoGroupId, ValueSign}; -use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; use ValueSign::{NonNegative, NonPositive, Unknown}; const TS: usize = 0; const VALUE: usize = 1; const HOST: usize = 2; -const MINUTE_MS: u64 = 60_000; +const MINUTE_MS: i64 = 60_000; /// The memo of every example: `latency.value` declared non-negative, and /// `requests_total.value` as Stage 0 declares a counter's. fn memo() -> Memo { - let non_negative = |source: &str| { - ( - SourceColumn { - source: source.into(), - column: "value".into(), - }, - NonNegative, - ) - }; - Memo::new(BTreeMap::from([ - non_negative("latency"), - non_negative("requests_total"), - ])) + let mut memo = Memo::default(); + for source in ["latency", "requests_total"] { + memo.declare_column_signs(source, BTreeMap::from([("value".into(), NonNegative)])); + } + memo } fn add(memo: &mut Memo, expression: Operator) -> MemoGroupId { @@ -63,40 +54,8 @@ fn depends(memo: &Memo, group: MemoGroupId) -> bool { } /// `Scan metric`: `[ts, value, host]`. -fn scan(metric: &str) -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("host", DataType::Utf8, false), - ], - 0, - vec![], - ), - }) -} - -fn range(child: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(MINUTE_MS), - kind: TimeRangeKind::Range, - child, - }) -} - -fn shift(at: Option, child: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 60_000, - at, - }, - child, - }) +fn series(metric: &str) -> Operator { + scan(metric, &[("host", DataType::Utf8)], vec![]) } fn compare( @@ -112,13 +71,6 @@ fn compare( } } -fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(pred), - child, - }) -} - /// `Project(ts, expr AS value, host)`. fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator { let item = |expr, alias: &str| ProjectItem { @@ -155,16 +107,6 @@ fn by_host() -> Reduction { Reduction::by(vec![HOST]) } -fn state(algorithm: SketchAlgorithm, child: MemoGroupId) -> Operator { - Operator::ASAP(ASAPOp::SummaryAgg { - child, - summary_type: FieldDataType::Sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }) -} - fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator { Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: state, @@ -188,8 +130,8 @@ fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator #[test] fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { let mut memo = memo(); - let g1 = add(&mut memo, scan("latency")); - let g2 = add(&mut memo, scan("gauge")); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, series("gauge")); let g3 = add( &mut memo, filter( @@ -201,8 +143,8 @@ fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { g1, ), ); - let g4 = add(&mut memo, range(g3)); - let g5 = add(&mut memo, shift(None, g4)); + let g4 = add(&mut memo, range(MINUTE_MS, g3)); + let g5 = add(&mut memo, shift(MINUTE_MS, None, g4)); let value = || Box::new(ScalarExpr::Column(VALUE)); let negated = || ScalarExpr::Negative { expr: value(), @@ -263,27 +205,27 @@ fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { #[test] fn aggregates_keep_their_keys_and_derive_their_values() { let mut memo = memo(); - let g1 = add(&mut memo, scan("latency")); + let g1 = add(&mut memo, series("latency")); let sum = || AggIntent::Sum { col: Some(VALUE) }; let g2 = add(&mut memo, aggregate(by_host(), sum(), g1)); let g3 = add( &mut memo, aggregate(by_host(), AggIntent::Max { col: Some(VALUE) }, g1), ); - let g4 = add(&mut memo, scan("gauge")); + let g4 = add(&mut memo, series("gauge")); let g5 = add(&mut memo, aggregate(by_host(), sum(), g4)); let g6 = add(&mut memo, aggregate(by_host(), AggIntent::Count, g4)); let g7 = add( &mut memo, aggregate(by_host(), AggIntent::Cardinality { cols: vec![VALUE] }, g4), ); - let requests = add(&mut memo, scan("requests_total")); - let g8 = add(&mut memo, range(requests)); + let requests = add(&mut memo, series("requests_total")); + let g8 = add(&mut memo, range(MINUTE_MS, requests)); let g9 = add( &mut memo, aggregate(Reduction::PerEntity, AggIntent::Rate, g8), ); - let g10 = add(&mut memo, range(g4)); + let g10 = add(&mut memo, range(MINUTE_MS, g4)); let g11 = add( &mut memo, aggregate(Reduction::PerEntity, AggIntent::Rate, g10), @@ -316,12 +258,18 @@ fn aggregates_keep_their_keys_and_derive_their_values() { #[test] fn states_record_their_input_sign_and_readouts_read_it() { let mut memo = memo(); - let g1 = add(&mut memo, scan("latency")); - let g2 = add(&mut memo, range(g1)); - let g3 = add(&mut memo, state(SketchAlgorithm::Kll, g2)); - let g4 = add(&mut memo, shift(None, g1)); - let g5 = add(&mut memo, range(g4)); - let g6 = add(&mut memo, state(SketchAlgorithm::Kll, g5)); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, range(MINUTE_MS, g1)); + let g3 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g2), + ); + let g4 = add(&mut memo, shift(MINUTE_MS, None, g1)); + let g5 = add(&mut memo, range(MINUTE_MS, g4)); + let g6 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g5), + ); let g7 = add( &mut memo, Operator::ASAP(ASAPOp::SummaryMerge { @@ -333,9 +281,12 @@ fn states_record_their_input_sign_and_readouts_read_it() { &mut memo, readout(SketchStatistic::Quantile { q: 0.99 }, g7), ); - let g9 = add(&mut memo, scan("gauge")); - let g10 = add(&mut memo, range(g9)); - let g11 = add(&mut memo, state(SketchAlgorithm::Hll, g10)); + let g9 = add(&mut memo, series("gauge")); + let g10 = add(&mut memo, range(MINUTE_MS, g9)); + let g11 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Hll), g10), + ); let g12 = add(&mut memo, readout(SketchStatistic::Cardinality, g11)); // [ts, state, host] for group in [g3, g6, g7] { @@ -368,11 +319,11 @@ fn a_top_k_ranks_values_of_its_inputs_sign() { ) }; let sum = AggIntent::Sum { col: Some(VALUE) }; - let gauge = add(&mut memo, scan("gauge")); + let gauge = add(&mut memo, series("gauge")); let gauge_sum = add(&mut memo, aggregate(by_host(), sum.clone(), gauge)); let gauge_top = top_5(&mut memo, gauge_sum); - let requests = add(&mut memo, scan("requests_total")); - let ranged = add(&mut memo, range(requests)); + let requests = add(&mut memo, series("requests_total")); + let ranged = add(&mut memo, range(MINUTE_MS, requests)); let rate = add( &mut memo, aggregate(Reduction::PerEntity, AggIntent::Rate, ranged), @@ -409,19 +360,29 @@ fn a_top_k_ranks_values_of_its_inputs_sign() { #[test] fn a_result_depends_on_the_evaluation_time_under_a_time_range_or_now() { let mut memo = memo(); - let g1 = add(&mut memo, scan("latency")); + let g1 = add(&mut memo, series("latency")); let g2 = add( &mut memo, aggregate(by_host(), AggIntent::Sum { col: Some(VALUE) }, g1), ); - let g3 = add(&mut memo, range(g1)); - let g4 = add(&mut memo, state(SketchAlgorithm::Kll, g3)); + let g3 = add(&mut memo, range(MINUTE_MS, g1)); + let g4 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g3), + ); let g5 = add( &mut memo, - shift(Some(AtModifier::Timestamp(1_700_000_000_000)), g3), + shift( + MINUTE_MS, + Some(AtModifier::Timestamp(1_700_000_000_000)), + g3, + ), + ); + let g6 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g5), ); - let g6 = add(&mut memo, state(SketchAlgorithm::Kll, g5)); - let g7 = add(&mut memo, shift(Some(AtModifier::End), g3)); + let g7 = add(&mut memo, shift(MINUTE_MS, Some(AtModifier::End), g3)); let g8 = add( &mut memo, filter( @@ -454,8 +415,8 @@ fn a_result_depends_on_the_evaluation_time_under_a_time_range_or_now() { #[test] fn a_group_reads_only_time_series_when_every_scan_below_reads_one() { let mut memo = memo(); - let series = add(&mut memo, scan("latency")); - let window = add(&mut memo, range(series)); + let series = add(&mut memo, series("latency")); + let window = add(&mut memo, range(MINUTE_MS, series)); let table = |name: &str| { Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { @@ -489,3 +450,82 @@ fn a_group_reads_only_time_series_when_every_scan_below_reads_one() { assert!(!reads(requests)); assert!(!reads(join)); } + +/// `count_values without (host) ("version", latency)`: a `without` +/// aggregate's layout adds no label column for `count_values` +/// (`aggregate_output_schema`), so the group has one sign per output +/// column, the count's non-negative. +#[test] +fn count_values_without_has_one_sign_per_output_column() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add( + &mut memo, + aggregate( + Reduction::Reduce(GroupKeys::without(vec![HOST])), + AggIntent::CountValues { + label: "version".into(), + }, + g1, + ), + ); + let group = &memo.groups()[g2.0 as usize].group_properties; + assert_eq!(group.column_sign.len(), group.schema.fields.len()); + assert_eq!(group.column_sign.last(), Some(&NonNegative)); +} + +/// A group's properties come from its operator and its input groups' +/// properties: the scan's schema, the state's kind and state column, the +/// readout's value column, and whether equal copies may be one group. +#[test] +fn group_properties_come_from_input_groups() { + let mut memo = Memo::default(); + let g1 = memo + .find_or_add_group(scan( + "latency", + &[ + ("job", DataType::Utf8), + ("a", DataType::Float64), + ("b", DataType::Float64), + ], + vec![], + )) + .unwrap(); + let api = compare( + ScalarExpr::Column(2), + CompareOpKind::Eq, + ScalarExpr::Literal(ScalarValue::Utf8("api".into())), + ); + let g2 = memo.find_or_add_group(filter(api, g1)).unwrap(); + let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); + let g4 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g3)) + .unwrap(); + let readout = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Quantile { q: 0.99 }, + })) + .unwrap(); + let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; + assert_eq!(properties(g1).schema.fields.len(), 5); + assert_eq!(properties(g1).aggregate_coverage, None); + assert_eq!(properties(g4).result_kind, OperatorResultKind::State); + assert!(properties(g4) + .schema + .fields + .iter() + .any(|field| field.name == "state" && !field.is_plain())); + assert!(properties(g4).aggregate_coverage.is_some()); + assert_eq!( + properties(readout).result_kind, + OperatorResultKind::InstantVector + ); + assert!(properties(readout) + .schema + .fields + .iter() + .any(|field| field.name == "quantile" + && field.dtype == FieldDataType::Plain(DataType::Float64))); + assert!(properties(g4).shareable_when_equal); +} diff --git a/crates/types/tests/memo_summary_coverage.rs b/crates/types/tests/memo_summary_coverage.rs index e2199edb1..c2e61eeb0 100644 --- a/crates/types/tests/memo_summary_coverage.rs +++ b/crates/types/tests/memo_summary_coverage.rs @@ -9,10 +9,11 @@ //! G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) //! ``` use std::ops::Bound; -use std::time::Duration; -use asap_types::ir::operator::arguments::{AtModifier, GroupKeys, Reduction, Source, TimeShift}; +use asap_types::ir::operator::arguments::{AtModifier, Reduction}; use asap_types::ir::operator::AggIntent; + +use crate::memo_fixtures::{filter, range, scan, shift, state}; use asap_types::ir::properties::summary_coverage::{ ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, SummaryCoverage, }; @@ -20,16 +21,13 @@ use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarVal use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, - SchemaDerivationError, TimeRangeKind, + ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, SchemaDerivationError, }; use asap_types::stages::{ CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, }; -use asap_types::summary_type::{ - ExactKind, MergeInputRule, SketchAlgorithm, SketchStatistic, SummaryUpdate, -}; +use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::definition_id; const TS: usize = 0; @@ -40,24 +38,16 @@ const MINUTE_MS: i64 = 60_000; const HOUR_MS: i64 = 60 * MINUTE_MS; /// `Scan latency{predicates}`: `[ts, value, job, a, b]`. -fn scan(predicates: Vec>) -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency".into(), - }, - predicates: predicates.into_iter().map(Predicate).collect(), - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, false), - Field::plain("a", DataType::Float64, false), - Field::plain("b", DataType::Float64, false), - ], - 0, - vec![], - ), - }) +fn latency(predicates: Vec>) -> Operator { + scan( + "latency", + &[ + ("job", DataType::Utf8), + ("a", DataType::Float64), + ("b", DataType::Float64), + ], + predicates, + ) } fn job_is(job: &str) -> ScalarExpr { @@ -84,40 +74,6 @@ fn a_plus_b_above_10() -> ScalarExpr { } } -fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(pred), - child, - }) -} - -/// `TimeShift(offset_ms, at)`: PromQL `offset` and `@`. -fn shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { offset_ms, at }, - child, - }) -} - -fn range(width_ms: i64, child: MemoGroupId) -> Operator { - Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms as u64), - kind: TimeRangeKind::Range, - child, - }) -} - -/// `SummaryAgg[summary_type]` of the sample value per series. -fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator { - Operator::ASAP(ASAPOp::SummaryAgg { - child, - summary_type, - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }) -} - fn kll_summary_type() -> FieldDataType { FieldDataType::Sketch(SketchAlgorithm::Kll) } @@ -204,58 +160,12 @@ fn time_of(memo: &Memo, group: MemoGroupId) -> Constraint { selection.columns()[&ts()].clone() } -/// A group's properties come from its operator and its input groups' -/// properties: the scan's schema, the state's kind and state column, the -/// readout's value column, and whether equal copies may be one group. -#[test] -fn group_properties_come_from_input_groups() { - let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); - let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); - let readout = memo - .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: g4, - query: SketchStatistic::Quantile { q: 0.99 }, - })) - .unwrap(); - let limit = memo - .find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { - n: Some(1), - offset: 0, - partition_by: GroupKeys::default(), - child: g1, - })) - .unwrap(); - let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; - assert_eq!(properties(g1).schema.fields.len(), 5); - assert_eq!(properties(g1).aggregate_coverage, None); - assert_eq!(properties(g4).result_kind, OperatorResultKind::State); - assert!(properties(g4) - .schema - .fields - .iter() - .any(|field| field.name == "state" && !field.is_plain())); - assert!(properties(g4).aggregate_coverage.is_some()); - assert_eq!( - properties(readout).result_kind, - OperatorResultKind::InstantVector - ); - assert!(properties(readout) - .schema - .fields - .iter() - .any(|field| field.name == "quantile" - && field.dtype == FieldDataType::Plain(DataType::Float64))); - assert!(properties(g4).shareable_when_equal); - assert!(!properties(limit).shareable_when_equal); -} - /// G4: `job = "api"` and the minute are lifted; the definition is the KLL /// over G1. #[test] fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let coverage = coverage(&memo, g4); assert_eq!(coverage.definition().input_group, Some(g1)); @@ -278,7 +188,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { #[test] fn states_with_equal_definitions_and_disjoint_selections_merge() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let g7 = kll_of_last_minute_where(&mut memo, g1, job_is("web")); assert_eq!( @@ -306,7 +216,7 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { #[test] fn states_with_overlapping_selections_do_not_merge() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let groups = memo.groups().len(); assert!(matches!( @@ -325,7 +235,7 @@ fn states_with_overlapping_selections_do_not_merge() { #[test] fn a_filter_that_is_no_selection_stays_unlifted() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let g10 = kll_of_last_minute_where( &mut memo, @@ -369,7 +279,7 @@ fn a_above(threshold: MemoGroupId) -> ScalarExpr { #[test] fn states_whose_definitions_read_the_same_subquery_merge() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let threshold = memo .find_or_add_group(Operator::NonASAP(NonASAPOp::Values { rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(10.0))]], @@ -409,7 +319,7 @@ fn states_whose_definitions_read_the_same_subquery_merge() { fn scan_predicates_lift_and_their_states_merge() { let mut memo = Memo::default(); let state_of_job = |memo: &mut Memo, job: &str| { - let series = memo.find_or_add_group(scan(vec![job_is(job)])).unwrap(); + let series = memo.find_or_add_group(latency(vec![job_is(job)])).unwrap(); let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); memo.find_or_add_group(kll(last_minute)).unwrap() }; @@ -419,7 +329,7 @@ fn scan_predicates_lift_and_their_states_merge() { assert_eq!(definition.input_group, None); assert_eq!( definition.unlifted_operators, - vec![kll_unlifted(), scan(vec![]).map_children(|_| None)] + vec![kll_unlifted(), latency(vec![]).map_children(|_| None)] ); assert_eq!( coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], @@ -447,7 +357,7 @@ fn scan_predicates_lift_and_their_states_merge() { #[test] fn a_merge_over_a_chains_windows_covers_the_direct_build() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(shift(HOUR_MS, None, g1)).unwrap(); let g3 = memo.find_or_add_group(range(5 * MINUTE_MS, g2)).unwrap(); let g4 = memo.find_or_add_group(kll(g3)).unwrap(); @@ -490,7 +400,7 @@ fn a_merge_over_a_chains_windows_covers_the_direct_build() { #[test] fn time_range_and_shifts_lift_into_one_interval_on_the_timestamp_column() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let plain = kll_over_last_5m(&mut memo, g1); let offset = kll_over_last_5m_of(&mut memo, shift(HOUR_MS, None, g1)); let at_t = kll_over_last_5m_of( @@ -540,7 +450,7 @@ fn kll_over_last_5m_of(memo: &mut Memo, expression: Operator) -> Me #[test] fn at_end_and_at_start_share_the_subtree_they_anchor_and_stay_unlifted() { let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); let g3 = memo.find_or_add_group(kll(g2)).unwrap(); assert_eq!(time_of(&memo, g3), relative(-5 * MINUTE_MS, 0)); @@ -571,7 +481,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { semantics: ExprSemantics::Sql, }; let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo .find_or_add_group(filter( ScalarExpr::BoolAnd(vec![ @@ -619,7 +529,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { fn counter_states_merge_over_windows_disjoint_in_time() { let increase = || FieldDataType::exact(ExactKind::Increase); let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); let g3 = memo.find_or_add_group(state(increase(), g2)).unwrap(); let g4 = memo @@ -660,7 +570,7 @@ fn an_exact_aggregate_has_coverage() { }) }; let mut memo = Memo::default(); - let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(filter(job_is("api"), g1)).unwrap(); let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); let g4 = memo.find_or_add_group(sum_by_job(g3)).unwrap(); From c71be033173097d28aa6aa0d953f6e3c80858439 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:13:53 +0000 Subject: [PATCH 03/12] refactor(types): aggregate coverage derivation beside the other group properties; tests of the aggregate layout (#1207 PR 2 review) - SummaryCoverage::derive_from_inputs and the walk helpers it uses move from ir/properties/summary_coverage.rs to the new stages/logical/aggregate_coverage.rs, next to column_properties.rs; the SummaryCoverage type and its other methods stay in ir/properties. The test of constraint_of moves with it. - aggregate_schema.rs tests aggregate_output_columns for by, without, count_values, absent and a per-entity reduction. --- .../src/ir/properties/summary_coverage.rs | 772 +---------------- .../types/src/ir/schema/aggregate_schema.rs | 95 +++ .../src/stages/logical/aggregate_coverage.rs | 774 ++++++++++++++++++ crates/types/src/stages/logical/memo/tests.rs | 5 +- crates/types/src/stages/logical/mod.rs | 1 + crates/types/src/stages/mod.rs | 1 - 6 files changed, 886 insertions(+), 762 deletions(-) create mode 100644 crates/types/src/stages/logical/aggregate_coverage.rs diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index f1734c116..349332a0b 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -3,17 +3,8 @@ //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! -//! Coverage is derived once for each aggregate memo group (a summary state -//! or an exact `Aggregate`), from its input groups, never declared, and -//! stored in its `GroupProperties::aggregate_coverage`. Walking down from a -//! `SummaryAgg` or `Aggregate` through the input groups' expressions, a -//! predicate conjunct moves into `selection` when it can be lifted to the -//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and -//! direct-column `Project` items) and it is a value set or an interval on -//! one column. The `TimeRange` and `TimeShift`s become one interval on the -//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in -//! `definition` as a residual, so two states are merged only when they -//! compute the same thing over disjoint rows. Node trees carry no coverage. +//! Coverage is derived once for each aggregate memo group, from its input +//! groups (`stages::logical::aggregate_coverage`); node trees carry none. use std::cmp::Ordering; use std::collections::BTreeMap; use std::ops::Bound; @@ -21,17 +12,11 @@ use std::ops::Bound; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::ir::operator::arguments::{AtModifier, Reduction}; -use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; -use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; -use crate::ir::operator::node::{Operator, OperatorResultKind}; -use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; -use crate::ir::scalar::{Predicate, ScalarExpr}; -use crate::ir::schema::{ColumnId, Schema}; -use crate::ir::SchemaDerivationError; -use crate::stages::{MemoGroupAsInput, MemoGroupId}; -use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; +use crate::ir::operator::node::Operator; +use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::schema::ColumnId; +use crate::stages::MemoGroupId; +use crate::summary_type::MergeInputRule; /// What a summary state computes, with every lifted selection removed. #[derive(Debug, Clone, PartialEq)] @@ -277,71 +262,7 @@ pub enum CoverageError { RegroupedBuild { reason: String }, } -/// A memo group read as an input, and the inputs a rule builds over such -/// groups. -type GroupInput<'m> = InputOrTemporaryBuild>; - impl SummaryCoverage { - /// Coverage of what a memo group computes with `operator` over its - /// input groups: a `SummaryAgg`, an exact `Aggregate`, a - /// `SummaryTumblingWindowChain` (a build over its input, the timestamp - /// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the - /// chain's coverage with the windows' interval on the timestamp column), - /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped - /// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` - /// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge - /// reads its inputs' stored coverage and fails - /// unless every input has the same definition and their selections are - /// pairwise disjoint; a family whose merge is idempotent - /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may - /// also merge overlapping selections, and a counter family - /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. - pub(crate) fn derive_from_inputs<'m>( - operator: &Operator>, - ) -> Result { - let as_built = |op: &ASAPOp>| -> ASAPOp> { - op.map_children(|input| InputOrTemporaryBuild::Input(*input)) - }; - match operator { - Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { - let top = - unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); - of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) - } - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - if operator.output_kind() == OperatorResultKind::State => - { - of_window_selection(operator) - } - Operator::NonASAP(_) => match selected_build(operator)? { - Some(build) => of_summary_agg(&build), - None => Err(CoverageError::NotSummary), - }, - Operator::ASAP( - op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), - ) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryMerge { - children, group_by, .. - }) => { - let inputs = children - .iter() - .map(|child| { - child - .aggregate_coverage() - .cloned() - .ok_or(CoverageError::NotSummary) - }) - .collect::, _>>()?; - let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; - match group_by { - None => Ok(merged), - Some(keys) => regrouped(merged, &children[0], keys), - } - } - Operator::ASAP(_) => Err(CoverageError::NotSummary), - } - } - /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, /// of its input group and unlifted operators. Two equal definitions /// (the equality a merge requires) of one memo get equal ids, so a @@ -358,659 +279,15 @@ impl SummaryCoverage { } } -/// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by -/// their key columns, the build that computes exactly the kept states: the -/// same build over its input rows filtered by the same predicate on the -/// input columns those keys name. `None` for any other operator. -fn selected_build<'m>( - operator: &Operator>, -) -> Result>>, CoverageError> { - let Operator::NonASAP(NonASAPOp::Filter { - pred, - child: states, - }) = operator - else { - return Ok(None); - }; - if operator.output_kind() != OperatorResultKind::State { - return Ok(None); - } - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction: Reduction::Reduce(by), - filter, - }) = states.operator() - else { - return Err(CoverageError::NotSummary); - }; - if by.is_without() { - return Err(CoverageError::NotSummary); - } - // Each state key column is the input column of the same name it groups by. - let mut unmapped = None; - let rows_pred = pred.0.map_columns(&mut |column| { - let mapped = by.keys().get(column).copied().filter(|&input_column| { - matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), - (Some(at_state), Some(at_input)) if at_state.name == at_input.name) - }); - mapped.unwrap_or_else(|| { - unmapped = Some(column); - column - }) - }); - if let Some(key) = unmapped { - return Err(CoverageError::NotAGroupKey { key }); - } - let mut built = |input: &MemoGroupAsInput<'m>| InputOrTemporaryBuild::Input(*input); - let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { - reason: error.to_string(), - }; - let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { - pred: Predicate(rows_pred.map_operator_refs(&mut built)), - child: InputOrTemporaryBuild::Input(child), - }) - .map_err(regrouped_build)?; - let reduction = Reduction::Reduce(by); - summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; - Ok(Some(ASAPOp::SummaryAgg { - child: rows, - summary_type: family, - input, - reduction, - filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), - })) -} - -/// The coverage of a merge of states covering `inputs`, whose builds read -/// rows with timestamp column `time_column`. -fn of_merge( - inputs: &[SummaryCoverage], - time_column: Option<&ColumnIdentity>, -) -> Result { - let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; - if inputs - .iter() - .any(|input| input.definition != first.definition) - { - return Err(CoverageError::DefinitionMismatch); - } - // A family without a merge is checked as the strictest rule; the - // executor rejects the merge itself. - let required = match first.definition.unlifted_operators.first() { - Some(Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - .. - })) => SummaryTypeCapability::of(family).merge_inputs, - _ => None, - } - .unwrap_or(MergeInputRule::Disjoint); - let separated = |a: &SelectionBox, b: &SelectionBox| match required { - MergeInputRule::MayOverlap => true, - // Consecutive windows of one series: disjoint on the timestamp column. - MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { - matches!( - (a.columns.get(column), b.columns.get(column)), - (Some(a), Some(b)) if a.disjoint(b) - ) - }), - MergeInputRule::Disjoint => a.disjoint(b), - }; - // Every pair of boxes of different inputs. - if required != MergeInputRule::MayOverlap - && inputs.iter().enumerate().any(|(i, input)| { - inputs[..i].iter().any(|other| { - input - .selection - .iter() - .any(|a| other.selection.iter().any(|b| !separated(a, b))) - }) - }) - { - return Err(CoverageError::InputsOverlap { required }); - } - Ok(SummaryCoverage { - definition: first.definition.clone(), - selection: union(inputs.iter().flat_map(|c| c.selection.clone()).collect()), - }) -} - -/// The coverage of a regrouping merge, keyed by `keys`, of states whose -/// merge covers `merged` (`first` is its first input): the same rows -/// (`selection`), computed by the input's build grouped by the columns -/// `keys` name. The input states of one output group are different groups -/// of one build, so their rows are disjoint. A counter family merges only -/// time-disjoint windows, never different groups at one time. -fn regrouped( - merged: SummaryCoverage, - first: &MemoGroupAsInput<'_>, - keys: &[ColumnId], -) -> Result { - let Some(Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - input, - reduction, - filter, - .. - })) = merged.definition.unlifted_operators.first() - else { - return Err(CoverageError::NotSummary); - }; - if !SummaryTypeCapability::merges_groups(family) { - return Err(CoverageError::SummaryTypeCannotRegroup); - } - let state = first.schema(); - let rows = build_input_schema(first).ok_or(CoverageError::NotSummary)?; - // The build's input column each key of the state names. - let mapped = keys - .iter() - .map(|&key| { - let column = match reduction { - Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), - // A per-series state keeps its input's columns in place. - Reduction::PerEntity => Some(key), - Reduction::Reduce(_) => None, - }; - column - .filter(|&column| { - let (Some(at_state), Some(at_input)) = - (state.fields.get(key), rows.fields.get(column)) - else { - return false; - }; - at_state.is_plain() - && at_state.name == at_input.name - && at_state.table == at_input.table - }) - .ok_or(CoverageError::NotAGroupKey { key }) - }) - .collect::, _>>()?; - let reduction = Reduction::by(mapped); - summary_agg_schema(&rows, family, &reduction).map_err(|error| { - CoverageError::RegroupedBuild { - reason: error.to_string(), - } - })?; - // A direct build's definition, so its definition id is the same. - let mut definition = merged.definition.clone(); - definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction, - filter: filter.clone(), - }); - Ok(SummaryCoverage { - definition, - selection: merged.selection, - }) -} - -/// The schema of the rows the build of summary state `state` reads (its -/// `SummaryAgg`'s input; one build over a chain's window reads its range of -/// the chain's input, which has the input's schema). `None` when `state` -/// is no summary state. -fn build_input_schema(state: &I) -> Option { - if let Some(states) = selected_from(state) { - return build_input_schema(&states); - } - match state.operator() { - Operator::ASAP( - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => Some(child.schema().clone()), - Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { - build_input_schema(children.first()?) - } - _ => None, - } -} - -/// The timestamp column of the rows the build of summary state `state` -/// reads ([`build_input_schema`]), as a selection names it. -fn time_column_of_build_input(state: &I) -> Option { - let rows = build_input_schema(state)?; - column_identity(&rows, rows.time_index?) -} - -/// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` -/// over memo groups (or over an input a rule built over them): a chain -/// covers what its `SummaryAgg` over the same input would. -fn of_summary_agg(build: &ASAPOp>) -> Result { - let (ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction, - filter, - } - | ASAPOp::SummaryTumblingWindowChain { - child, - summary_type: family, - input, - reduction, - filter, - .. - }) = build - else { - unreachable!("called on a SummaryAgg or a chain"); - }; - of_aggregate(child, filter.as_ref(), |residual_filter| { - Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - filter: residual_filter, - }) - }) -} - -/// The coverage of an aggregate over `child`: its input's operators walked -/// down through the liftable operators of the input groups' expressions. -/// `filter` is the aggregate's own row filter (a `SummaryAgg`'s), and -/// `unlifted_top` makes the aggregate's unlifted operator from that filter's residual. -fn of_aggregate( - child: &GroupInput<'_>, - filter: Option<&Predicate>>, - unlifted_top: impl FnOnce(Option>>) -> Operator>, -) -> Result { - // The operators a predicate can be lifted through, top to bottom, and - // the first input below them. A subquery's group stays named in the - // residual of the operator that reads it. - let mut chain: Vec>> = Vec::new(); - let mut base = child.clone(); - loop { - let operator = base.operator(); - let passes = match &operator { - Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, - Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, - // `@ end()` / `@ start()` pin their subtree to one end of the - // request: not lifted, so their group is the input group. - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - !matches!(shift.at, Some(AtModifier::Start | AtModifier::End)) - } - _ => false, - }; - if !passes { - break; - } - // The main input is an operator's first child. - base = operator.children()[0].clone(); - chain.push(operator); - } - let InputOrTemporaryBuild::Input(input_group) = &base else { - unreachable!("a rule builds only liftable operators over memo groups"); - }; - let top = child.schema(); - // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual - // of `chain[d]` when it is a `Filter`. - let mut lifted = BTreeMap::new(); - let agg_filter = filter.and_then(|pred| residual(pred, top, &[], &mut lifted)); - let kept: Vec>>> = chain - .iter() - .enumerate() - .map(|(depth, link)| match link { - Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { - residual(pred, top, &chain[..depth], &mut lifted) - } - _ => None, - }) - .collect(); - // A `Scan`'s predicates (PromQL label matchers) are lifted too; when - // one is, the `Scan` with the rest of its predicates is the last - // unlifted operator, and there is no input group. - let scan = match base.operator() { - Operator::NonASAP(NonASAPOp::Scan { - source, - predicates, - schema, - }) => { - let lifted_above = lifted.clone(); - let kept = predicates - .iter() - .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) - .collect(); - (lifted != lifted_above).then_some(NonASAPOp::Scan { - source, - predicates: kept, - schema, - }) - } - _ => None, - }; - - // The `TimeRange` and `TimeShift`s: one interval on the timestamp column - // of the `TimeRange`'s input, when they make one and it combines with - // what is lifted on that column already; else they stay unlifted. - let time = time_constraint(&chain).and_then(|constraint| { - let depth = chain - .iter() - .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; - let column = chain[depth].children()[0].schema().time_index?; - let column = column_at_top(column, &chain[..=depth])?; - column_identity(top, column)?; - Some((column, combined_with(lifted.get(&column), constraint)?)) - }); - let lift_time = time.is_some(); - lifted.extend(time); - - let mut unlifted_operators = vec![unlifted_top(agg_filter)]; - for (depth, link) in chain.iter().enumerate() { - match link { - Operator::NonASAP(NonASAPOp::Filter { .. }) => { - if let Some(rest) = kept[depth].clone() { - unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { - pred: rest, - child: None, - })); - } - } - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - if lift_time => {} - other => unlifted_operators.push(unlifted(other)), - } - } - let input_group = match scan { - Some(scan) => { - unlifted_operators.push(Operator::NonASAP(scan)); - None - } - None => Some(input_group.group_id), - }; - - let selection = SelectionBox { - columns: lifted - .into_iter() - .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) - .collect(), - }; - Ok(SummaryCoverage { - definition: SummaryStateDefinition { - input_group, - unlifted_operators, - }, - selection: vec![selection], - }) -} - -/// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest, each scalar subquery -/// naming the group it reads. `above` -/// are the operators between the predicate and the `SummaryAgg`, top to -/// bottom. A column [`column_identity`] cannot name stays. -fn residual( - pred: &Predicate>, - top: &Schema, - above: &[Operator>], - lifted: &mut BTreeMap, -) -> Option>> { - let mut rest: Vec>> = Vec::new(); - for conjunct in pred.0.conjuncts() { - let lift = constraint_of(conjunct).and_then(|(column, constraint)| { - let column = column_at_top(column, above)?; - column_identity(top, column)?; - Some((column, combined_with(lifted.get(&column), constraint)?)) - }); - match lift { - Some((column, constraint)) => { - lifted.insert(column, constraint); - } - None => rest.push(conjunct.map_operator_refs(&mut group_id)), - } - } - match rest.len() { - 0 => None, - 1 => rest.pop().map(Predicate), - _ => Some(Predicate(ScalarExpr::BoolAnd(rest))), - } -} - -/// `constraint` combined with what is lifted on its column already -/// (`existing`); `None` when the two do not combine. -fn combined_with(existing: Option<&Constraint>, constraint: Constraint) -> Option { - match existing { - Some(existing) => existing.intersect(&constraint), - None => Some(constraint), - } -} - -/// The memo group `input` is; `None` for a rule's temporary build. -fn group_id(input: &GroupInput<'_>) -> Option { - match input { - InputOrTemporaryBuild::Input(group) => Some(group.group_id), - InputOrTemporaryBuild::TemporaryBuild(_) => None, - } -} - -/// `link`, a liftable operator or an exact `Aggregate` kept whole in a -/// definition: its main input `None`, every other input the group it is. -fn unlifted(link: &Operator>) -> Operator> { - let mut operator = link.map_children(group_id); - match &mut operator { - Operator::NonASAP( - NonASAPOp::Filter { child, .. } - | NonASAPOp::Project { child, .. } - | NonASAPOp::TimeRange { child, .. } - | NonASAPOp::TimeShift { child, .. } - | NonASAPOp::Aggregate { child, .. }, - ) => *child = None, - _ => unreachable!("only liftable operators and aggregates are unlifted"), - } - operator -} - -/// Map a column of the input of `above.last()` up to the agg child's output: -/// a `Project` passes it only as a direct column item. -fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { - for link in above.iter().rev() { - if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { - column = cols - .iter() - .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; - } - } - Some(column) -} - -/// The identity of `schema`'s column `column`; `None` when its -/// `(table, name)` is not unique in `schema`, so a selection cannot name it. -fn column_identity(schema: &Schema, column: ColumnId) -> Option { - let field = schema.fields.get(column)?; - let namesakes = schema - .fields - .iter() - .filter(|f| f.name == field.name && f.table == field.table) - .count(); - (namesakes == 1).then(|| ColumnIdentity { - table: field.table.clone(), - name: field.name.clone(), - }) -} - -/// The interval on the timestamp column that `links` (top down, the -/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: -/// `(anchor − o − W, anchor − o]` for the one range `TimeRange` of width W -/// and the shifts' offsets summing to o, left-open and right-closed like a -/// PromQL range. The anchor is the evaluation time t -/// (`EvaluationTimePlusMs`), or T for `@ T` (`FixedValue(Timestamp)`). -/// `None` when they are not one interval: no or several range `TimeRange`s, -/// an instant one, several `@`s, or an `@ end()` or `@ start()`, which pins -/// its subtree to one end of the request and is never lifted (the front -/// ends put it above the subtree it anchors). -fn time_constraint(links: &[Operator]) -> Option { - let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); - for link in links { - match link { - Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { - if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { - return None; - } - } - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - offset_ms = offset_ms.checked_add(shift.offset_ms)?; - if let Some(anchor) = shift.at { - if at.replace(anchor).is_some() { - return None; - } - } - } - _ => {} - } - } - let width_ms = i64::try_from(width?.as_millis()).ok()?; - let fixed_anchor = match at { - Some(AtModifier::Timestamp(ms)) => Some(ms), - Some(AtModifier::Start | AtModifier::End) => return None, - None => None, - }; - let before_anchor = |ms: i64| -> Option { - Some(match fixed_anchor { - Some(anchor) => { - IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) - } - None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), - }) - }; - Some(Constraint::Interval { - lower: Bound::Excluded(before_anchor(offset_ms.checked_add(width_ms)?)?), - upper: Bound::Included(before_anchor(offset_ms)?), - }) -} - -/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain -/// (through more of them): the chain's coverage with the interval they -/// select on the timestamp column of the chain's input, in every box. -fn of_window_selection( - selection: &Operator>, -) -> Result { - let mut links = vec![selection.clone()]; - let mut below = *selection.children()[0]; - while let Some(child) = selected_from(&below) { - links.push(below.operator()); - below = child; - } - let only_time_selections = links.iter().all(|link| { - matches!( - link, - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - ) - }); - if !only_time_selections - || !matches!( - below.operator(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - { - return Err(CoverageError::NotSummary); - } - let chain = below - .aggregate_coverage() - .ok_or(CoverageError::NotSummary)?; - let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NotSummary)?; - let constraint = time_constraint(&links).ok_or(CoverageError::NotSummary)?; - let selection = chain - .selection - .iter() - .map(|selection_box| { - let mut columns = selection_box.columns.clone(); - let combined = combined_with(columns.get(&time_column), constraint.clone()) - .ok_or(CoverageError::NotSummary)?; - columns.insert(time_column.clone(), combined); - Ok(SelectionBox { columns }) - }) - .collect::>()?; - Ok(SummaryCoverage { - definition: chain.definition.clone(), - selection, - }) -} - -/// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of -/// equalities on one column, with non-null literals. -fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { - let literal = |e: &ScalarExpr| match e { - ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), - _ => None, - }; - match expr { - ScalarExpr::Compare { - left, op, right, .. - } => { - let (column, value, op) = match (left.as_ref(), right.as_ref()) { - (ScalarExpr::Column(c), r) => (*c, literal(r)?, op.clone()), - (l, ScalarExpr::Column(c)) => (*c, literal(l)?, flip(op)?), - _ => return None, - }; - let constraint = match op { - CompareOpKind::Eq => Constraint::In(vec![value]), - CompareOpKind::Ne => Constraint::NotIn(vec![value]), - CompareOpKind::Lt => interval(Bound::Unbounded, Bound::Excluded(value)), - CompareOpKind::Le => interval(Bound::Unbounded, Bound::Included(value)), - CompareOpKind::Gt => interval(Bound::Excluded(value), Bound::Unbounded), - CompareOpKind::Ge => interval(Bound::Included(value), Bound::Unbounded), - _ => return None, - }; - Some((column, constraint)) - } - ScalarExpr::InList { - expr, - list, - negated, - } => { - let ScalarExpr::Column(column) = expr.as_ref() else { - return None; - }; - let values = list.iter().map(literal).collect::>>()?; - let values = dedup(values); - Some(( - *column, - if *negated { - Constraint::NotIn(values) - } else { - Constraint::In(values) - }, - )) - } - ScalarExpr::BoolOr(disjuncts) => { - let mut column = None; - let mut values = Vec::new(); - for disjunct in disjuncts { - let (c, Constraint::In(v)) = constraint_of(disjunct)? else { - return None; - }; - if column.replace(c).is_some_and(|prev| prev != c) { - return None; - } - values.extend(v); - } - Some((column?, Constraint::In(dedup(values)))) - } - _ => None, - } -} - -fn flip(op: &CompareOpKind) -> Option { - Some(match op { - CompareOpKind::Eq => CompareOpKind::Eq, - CompareOpKind::Ne => CompareOpKind::Ne, - CompareOpKind::Lt => CompareOpKind::Gt, - CompareOpKind::Le => CompareOpKind::Ge, - CompareOpKind::Gt => CompareOpKind::Lt, - CompareOpKind::Ge => CompareOpKind::Le, - _ => return None, - }) -} - /// An interval between two fixed values. -fn interval(lower: Bound, upper: Bound) -> Constraint { +pub(crate) fn interval(lower: Bound, upper: Bound) -> Constraint { Constraint::Interval { lower: lower.map(IntervalConstraintLimit::FixedValue), upper: upper.map(IntervalConstraintLimit::FixedValue), } } -fn dedup(values: Vec) -> Vec { +pub(crate) fn dedup(values: Vec) -> Vec { let mut out: Vec = Vec::new(); for v in values { if !out.contains(&v) { @@ -1148,7 +425,7 @@ fn tighter( impl Constraint { /// The conjunction of two constraints on one column, when it is again a /// single constraint. - fn intersect(&self, other: &Self) -> Option { + pub(crate) fn intersect(&self, other: &Self) -> Option { use Constraint::*; Some(match (self, other) { (In(a), In(b)) => In(filter_values(a, b, true)?), @@ -1181,7 +458,7 @@ impl Constraint { } /// Whether no value satisfies both; `false` when unsure. - fn disjoint(&self, other: &Self) -> bool { + pub(crate) fn disjoint(&self, other: &Self) -> bool { use Constraint::*; match (self, other) { (In(a), In(b)) => a.iter().all(|v| among(v, b) == Some(false)), @@ -1219,7 +496,7 @@ impl SelectionBox { impl SelectionBox { /// Whether no row lies in both boxes: some column is restricted by both /// and the restrictions are disjoint. - fn disjoint(&self, other: &Self) -> bool { + pub(crate) fn disjoint(&self, other: &Self) -> bool { self.columns .iter() .any(|(column, a)| other.columns.get(column).is_some_and(|b| a.disjoint(b))) @@ -1229,7 +506,7 @@ impl SelectionBox { /// Join boxes that differ only in one column whose union is again one /// constraint: touching value ranges, or the values of one `In` column. /// Gaps stay separate boxes. -fn union(mut boxes: Vec) -> Vec { +pub(crate) fn union(mut boxes: Vec) -> Vec { let mut i = 0; while i < boxes.len() { let joined = (i + 1..boxes.len()).find_map(|j| join(&boxes[i], &boxes[j]).map(|b| (j, b))); @@ -1309,7 +586,7 @@ fn touching( #[cfg(test)] mod tests { use super::*; - use crate::ir::scalar::ExprSemantics; + use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { @@ -1319,27 +596,6 @@ mod tests { } } - /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. - #[test] - fn a_value_comparison_is_an_interval_between_fixed_values() { - let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(0)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), - semantics: ExprSemantics::Sql, - }; - assert_eq!( - constraint_of(&price_gt_10), - Some(( - 0, - Constraint::Interval { - lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), - upper: Bound::Unbounded, - } - )) - ); - } - /// `m[5m]` and `m[5m] offset 5m` are disjoint and touch; neither /// compares with an `@ T` interval. #[test] diff --git a/crates/types/src/ir/schema/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs index cf8e920d4..345cb78cb 100644 --- a/crates/types/src/ir/schema/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -318,6 +318,101 @@ fn without_output_schema( mod tests { use super::*; + /// `[ts, value, job, host]`, the time series schema the layout tests read. + fn series() -> Schema { + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp, false), + col("value", DataType::Float64, false), + col("job", DataType::Utf8, false), + col("host", DataType::Utf8, false), + ], + 0, + vec![], + ) + } + + /// Each output column's source, and one source per column of + /// `aggregate_output_schema`'s output. + fn columns(reduction: Reduction, measures: &[AggIntent]) -> Vec { + let sources = aggregate_output_columns(&series(), &reduction, measures).unwrap(); + let schema = aggregate_output_schema(&series(), &reduction, measures, &[]).unwrap(); + assert_eq!(sources.len(), schema.fields.len()); + sources + } + + /// `sum by (job)`: the key, then the measure. + #[test] + fn a_by_aggregate_keeps_its_keys_then_its_measures() { + use OutputColumnSource::*; + assert_eq!( + columns(Reduction::by(vec![2]), &[AggIntent::Sum { col: None }]), + vec![InputColumn(2), Measure(0)] + ); + } + + /// `sum without (host)`: every label but `host`, then the measure. + #[test] + fn a_without_aggregate_keeps_the_other_labels_then_its_measures() { + use OutputColumnSource::*; + assert_eq!( + columns( + Reduction::Reduce(GroupKeys::without(vec![3])), + &[AggIntent::Sum { col: None }] + ), + vec![InputColumn(2), Measure(0)] + ); + } + + /// `count_values by (job) ("version", v)` adds the `version` label before + /// the count; `count_values without (host)` adds none, like its schema. + #[test] + fn count_values_adds_its_label_only_by_keys() { + use OutputColumnSource::*; + let count_values = AggIntent::CountValues { + label: "version".into(), + }; + assert_eq!( + columns(Reduction::by(vec![2]), std::slice::from_ref(&count_values)), + vec![InputColumn(2), AddedLabel, Measure(0)] + ); + assert_eq!( + columns( + Reduction::Reduce(GroupKeys::without(vec![3])), + &[count_values] + ), + vec![InputColumn(2), Measure(0)] + ); + } + + /// `absent(m{job="a", host="b"})` adds one label per matcher before its + /// value. + #[test] + fn absent_adds_its_labels_before_its_value() { + use OutputColumnSource::*; + let absent = AggIntent::Absent { + labels: std::collections::BTreeMap::from([ + ("host".to_string(), "b".to_string()), + ("job".to_string(), "a".to_string()), + ]), + }; + assert_eq!( + columns(Reduction::by(vec![]), &[absent]), + vec![AddedLabel, AddedLabel, Measure(0)] + ); + } + + /// A per-entity reduction keeps every input column, the value replaced + /// by the measure. + #[test] + fn a_per_entity_reduction_replaces_the_value_by_the_measure() { + use OutputColumnSource::*; + assert_eq!( + columns(Reduction::PerEntity, &[AggIntent::Rate]), + vec![InputColumn(0), Measure(0), InputColumn(2), InputColumn(3)] + ); + } + fn col(name: &str, dtype: DataType, nullable: bool) -> Field { Field::plain(name, dtype, nullable) } diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs new file mode 100644 index 000000000..18accf1e5 --- /dev/null +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -0,0 +1,774 @@ +//! Aggregate coverage, a group property: [`SummaryCoverage`] derived once +//! for each aggregate memo group from its input groups. +//! +//! It is derived for a summary state or an exact `Aggregate`, never +//! declared, and +//! stored in its `GroupProperties::aggregate_coverage`. Walking down from a +//! `SummaryAgg` or `Aggregate` through the input groups' expressions, a +//! predicate conjunct moves into `selection` when it can be lifted to the +//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and +//! direct-column `Project` items) and it is a value set or an interval on +//! one column. The `TimeRange` and `TimeShift`s become one interval on the +//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! `definition` as a residual, so two states are merged only when they +//! compute the same thing over disjoint rows. Node trees carry no coverage. + +use std::collections::BTreeMap; +use std::ops::Bound; + +use super::MemoGroupAsInput; +use crate::ir::operator::arguments::{AtModifier, Reduction}; +use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; +use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; +use crate::ir::operator::node::{Operator, OperatorResultKind}; +use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; +use crate::ir::properties::summary_coverage::{ + dedup, interval, union, ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, + SelectionBox, SummaryCoverage, SummaryStateDefinition, +}; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::{Predicate, ScalarExpr}; +use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::SchemaDerivationError; +use crate::stages::MemoGroupId; +use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; + +impl SummaryCoverage { + /// Coverage of what a memo group computes with `operator` over its + /// input groups: a `SummaryAgg`, an exact `Aggregate`, a + /// `SummaryTumblingWindowChain` (a build over its input, the timestamp + /// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the + /// chain's coverage with the windows' interval on the timestamp column), + /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped + /// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` + /// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge + /// reads its inputs' stored coverage and fails + /// unless every input has the same definition and their selections are + /// pairwise disjoint; a family whose merge is idempotent + /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may + /// also merge overlapping selections, and a counter family + /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. + pub(crate) fn derive_from_inputs<'m>( + operator: &Operator>, + ) -> Result { + let as_built = |op: &ASAPOp>| -> ASAPOp> { + op.map_children(|input| InputOrTemporaryBuild::Input(*input)) + }; + match operator { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = + unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); + of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if operator.output_kind() == OperatorResultKind::State => + { + of_window_selection(operator) + } + Operator::NonASAP(_) => match selected_build(operator)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NotSummary), + }, + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => of_summary_agg(&as_built(op)), + Operator::ASAP(ASAPOp::SummaryMerge { + children, group_by, .. + }) => { + let inputs = children + .iter() + .map(|child| { + child + .aggregate_coverage() + .cloned() + .ok_or(CoverageError::NotSummary) + }) + .collect::, _>>()?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; + match group_by { + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), + } + } + Operator::ASAP(_) => Err(CoverageError::NotSummary), + } + } +} + +/// A memo group read as an input, and the inputs a rule builds over such +/// groups. +type GroupInput<'m> = InputOrTemporaryBuild>; + +/// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by +/// their key columns, the build that computes exactly the kept states: the +/// same build over its input rows filtered by the same predicate on the +/// input columns those keys name. `None` for any other operator. +fn selected_build<'m>( + operator: &Operator>, +) -> Result>>, CoverageError> { + let Operator::NonASAP(NonASAPOp::Filter { + pred, + child: states, + }) = operator + else { + return Ok(None); + }; + if operator.output_kind() != OperatorResultKind::State { + return Ok(None); + } + let Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: family, + input, + reduction: Reduction::Reduce(by), + filter, + }) = states.operator() + else { + return Err(CoverageError::NotSummary); + }; + if by.is_without() { + return Err(CoverageError::NotSummary); + } + // Each state key column is the input column of the same name it groups by. + let mut unmapped = None; + let rows_pred = pred.0.map_columns(&mut |column| { + let mapped = by.keys().get(column).copied().filter(|&input_column| { + matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), + (Some(at_state), Some(at_input)) if at_state.name == at_input.name) + }); + mapped.unwrap_or_else(|| { + unmapped = Some(column); + column + }) + }); + if let Some(key) = unmapped { + return Err(CoverageError::NotAGroupKey { key }); + } + let mut built = |input: &MemoGroupAsInput<'m>| InputOrTemporaryBuild::Input(*input); + let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { + reason: error.to_string(), + }; + let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { + pred: Predicate(rows_pred.map_operator_refs(&mut built)), + child: InputOrTemporaryBuild::Input(child), + }) + .map_err(regrouped_build)?; + let reduction = Reduction::Reduce(by); + summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; + Ok(Some(ASAPOp::SummaryAgg { + child: rows, + summary_type: family, + input, + reduction, + filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), + })) +} + +/// The coverage of a merge of states covering `inputs`, whose builds read +/// rows with timestamp column `time_column`. +fn of_merge( + inputs: &[SummaryCoverage], + time_column: Option<&ColumnIdentity>, +) -> Result { + let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; + if inputs + .iter() + .any(|input| input.definition != first.definition) + { + return Err(CoverageError::DefinitionMismatch); + } + // A family without a merge is checked as the strictest rule; the + // executor rejects the merge itself. + let required = match first.definition.unlifted_operators.first() { + Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + })) => SummaryTypeCapability::of(family).merge_inputs, + _ => None, + } + .unwrap_or(MergeInputRule::Disjoint); + let separated = |a: &SelectionBox, b: &SelectionBox| match required { + MergeInputRule::MayOverlap => true, + // Consecutive windows of one series: disjoint on the timestamp column. + MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { + matches!( + (a.columns.get(column), b.columns.get(column)), + (Some(a), Some(b)) if a.disjoint(b) + ) + }), + MergeInputRule::Disjoint => a.disjoint(b), + }; + // Every pair of boxes of different inputs. + if required != MergeInputRule::MayOverlap + && inputs.iter().enumerate().any(|(i, input)| { + inputs[..i].iter().any(|other| { + input + .selection + .iter() + .any(|a| other.selection.iter().any(|b| !separated(a, b))) + }) + }) + { + return Err(CoverageError::InputsOverlap { required }); + } + Ok(SummaryCoverage { + definition: first.definition.clone(), + selection: union(inputs.iter().flat_map(|c| c.selection.clone()).collect()), + }) +} + +/// The coverage of a regrouping merge, keyed by `keys`, of states whose +/// merge covers `merged` (`first` is its first input): the same rows +/// (`selection`), computed by the input's build grouped by the columns +/// `keys` name. The input states of one output group are different groups +/// of one build, so their rows are disjoint. A counter family merges only +/// time-disjoint windows, never different groups at one time. +fn regrouped( + merged: SummaryCoverage, + first: &MemoGroupAsInput<'_>, + keys: &[ColumnId], +) -> Result { + let Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + input, + reduction, + filter, + .. + })) = merged.definition.unlifted_operators.first() + else { + return Err(CoverageError::NotSummary); + }; + if !SummaryTypeCapability::merges_groups(family) { + return Err(CoverageError::SummaryTypeCannotRegroup); + } + let state = first.schema(); + let rows = build_input_schema(first).ok_or(CoverageError::NotSummary)?; + // The build's input column each key of the state names. + let mapped = keys + .iter() + .map(|&key| { + let column = match reduction { + Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), + // A per-series state keeps its input's columns in place. + Reduction::PerEntity => Some(key), + Reduction::Reduce(_) => None, + }; + column + .filter(|&column| { + let (Some(at_state), Some(at_input)) = + (state.fields.get(key), rows.fields.get(column)) + else { + return false; + }; + at_state.is_plain() + && at_state.name == at_input.name + && at_state.table == at_input.table + }) + .ok_or(CoverageError::NotAGroupKey { key }) + }) + .collect::, _>>()?; + let reduction = Reduction::by(mapped); + summary_agg_schema(&rows, family, &reduction).map_err(|error| { + CoverageError::RegroupedBuild { + reason: error.to_string(), + } + })?; + // A direct build's definition, so its definition id is the same. + let mut definition = merged.definition.clone(); + definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), + input: input.clone(), + reduction, + filter: filter.clone(), + }); + Ok(SummaryCoverage { + definition, + selection: merged.selection, + }) +} + +/// The schema of the rows the build of summary state `state` reads (its +/// `SummaryAgg`'s input; one build over a chain's window reads its range of +/// the chain's input, which has the input's schema). `None` when `state` +/// is no summary state. +fn build_input_schema(state: &I) -> Option { + if let Some(states) = selected_from(state) { + return build_input_schema(&states); + } + match state.operator() { + Operator::ASAP( + ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, + ) => Some(child.schema().clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { + build_input_schema(children.first()?) + } + _ => None, + } +} + +/// The timestamp column of the rows the build of summary state `state` +/// reads ([`build_input_schema`]), as a selection names it. +fn time_column_of_build_input(state: &I) -> Option { + let rows = build_input_schema(state)?; + column_identity(&rows, rows.time_index?) +} + +/// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` +/// over memo groups (or over an input a rule built over them): a chain +/// covers what its `SummaryAgg` over the same input would. +fn of_summary_agg(build: &ASAPOp>) -> Result { + let (ASAPOp::SummaryAgg { + child, + summary_type: family, + input, + reduction, + filter, + } + | ASAPOp::SummaryTumblingWindowChain { + child, + summary_type: family, + input, + reduction, + filter, + .. + }) = build + else { + unreachable!("called on a SummaryAgg or a chain"); + }; + of_aggregate(child, filter.as_ref(), |residual_filter| { + Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + filter: residual_filter, + }) + }) +} + +/// The coverage of an aggregate over `child`: its input's operators walked +/// down through the liftable operators of the input groups' expressions. +/// `filter` is the aggregate's own row filter (a `SummaryAgg`'s), and +/// `unlifted_top` makes the aggregate's unlifted operator from that filter's residual. +fn of_aggregate( + child: &GroupInput<'_>, + filter: Option<&Predicate>>, + unlifted_top: impl FnOnce(Option>>) -> Operator>, +) -> Result { + // The operators a predicate can be lifted through, top to bottom, and + // the first input below them. A subquery's group stays named in the + // residual of the operator that reads it. + let mut chain: Vec>> = Vec::new(); + let mut base = child.clone(); + loop { + let operator = base.operator(); + let passes = match &operator { + Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, + // `@ end()` / `@ start()` pin their subtree to one end of the + // request: not lifted, so their group is the input group. + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + !matches!(shift.at, Some(AtModifier::Start | AtModifier::End)) + } + _ => false, + }; + if !passes { + break; + } + // The main input is an operator's first child. + base = operator.children()[0].clone(); + chain.push(operator); + } + let InputOrTemporaryBuild::Input(input_group) = &base else { + unreachable!("a rule builds only liftable operators over memo groups"); + }; + let top = child.schema(); + // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual + // of `chain[d]` when it is a `Filter`. + let mut lifted = BTreeMap::new(); + let agg_filter = filter.and_then(|pred| residual(pred, top, &[], &mut lifted)); + let kept: Vec>>> = chain + .iter() + .enumerate() + .map(|(depth, link)| match link { + Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { + residual(pred, top, &chain[..depth], &mut lifted) + } + _ => None, + }) + .collect(); + // A `Scan`'s predicates (PromQL label matchers) are lifted too; when + // one is, the `Scan` with the rest of its predicates is the last + // unlifted operator, and there is no input group. + let scan = match base.operator() { + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) => { + let lifted_above = lifted.clone(); + let kept = predicates + .iter() + .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) + .collect(); + (lifted != lifted_above).then_some(NonASAPOp::Scan { + source, + predicates: kept, + schema, + }) + } + _ => None, + }; + + // The `TimeRange` and `TimeShift`s: one interval on the timestamp column + // of the `TimeRange`'s input, when they make one and it combines with + // what is lifted on that column already; else they stay unlifted. + let time = time_constraint(&chain).and_then(|constraint| { + let depth = chain + .iter() + .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; + let column = chain[depth].children()[0].schema().time_index?; + let column = column_at_top(column, &chain[..=depth])?; + column_identity(top, column)?; + Some((column, combined_with(lifted.get(&column), constraint)?)) + }); + let lift_time = time.is_some(); + lifted.extend(time); + + let mut unlifted_operators = vec![unlifted_top(agg_filter)]; + for (depth, link) in chain.iter().enumerate() { + match link { + Operator::NonASAP(NonASAPOp::Filter { .. }) => { + if let Some(rest) = kept[depth].clone() { + unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { + pred: rest, + child: None, + })); + } + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if lift_time => {} + other => unlifted_operators.push(unlifted(other)), + } + } + let input_group = match scan { + Some(scan) => { + unlifted_operators.push(Operator::NonASAP(scan)); + None + } + None => Some(input_group.group_id), + }; + + let selection = SelectionBox { + columns: lifted + .into_iter() + .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) + .collect(), + }; + Ok(SummaryCoverage { + definition: SummaryStateDefinition { + input_group, + unlifted_operators, + }, + selection: vec![selection], + }) +} + +/// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of +/// `top`, the agg child's schema) and return the rest, each scalar subquery +/// naming the group it reads. `above` +/// are the operators between the predicate and the `SummaryAgg`, top to +/// bottom. A column [`column_identity`] cannot name stays. +fn residual( + pred: &Predicate>, + top: &Schema, + above: &[Operator>], + lifted: &mut BTreeMap, +) -> Option>> { + let mut rest: Vec>> = Vec::new(); + for conjunct in pred.0.conjuncts() { + let lift = constraint_of(conjunct).and_then(|(column, constraint)| { + let column = column_at_top(column, above)?; + column_identity(top, column)?; + Some((column, combined_with(lifted.get(&column), constraint)?)) + }); + match lift { + Some((column, constraint)) => { + lifted.insert(column, constraint); + } + None => rest.push(conjunct.map_operator_refs(&mut group_id)), + } + } + match rest.len() { + 0 => None, + 1 => rest.pop().map(Predicate), + _ => Some(Predicate(ScalarExpr::BoolAnd(rest))), + } +} + +/// `constraint` combined with what is lifted on its column already +/// (`existing`); `None` when the two do not combine. +fn combined_with(existing: Option<&Constraint>, constraint: Constraint) -> Option { + match existing { + Some(existing) => existing.intersect(&constraint), + None => Some(constraint), + } +} + +/// The memo group `input` is; `None` for a rule's temporary build. +fn group_id(input: &GroupInput<'_>) -> Option { + match input { + InputOrTemporaryBuild::Input(group) => Some(group.group_id), + InputOrTemporaryBuild::TemporaryBuild(_) => None, + } +} + +/// `link`, a liftable operator or an exact `Aggregate` kept whole in a +/// definition: its main input `None`, every other input the group it is. +fn unlifted(link: &Operator>) -> Operator> { + let mut operator = link.map_children(group_id); + match &mut operator { + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::Project { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. } + | NonASAPOp::Aggregate { child, .. }, + ) => *child = None, + _ => unreachable!("only liftable operators and aggregates are unlifted"), + } + operator +} + +/// Map a column of the input of `above.last()` up to the agg child's output: +/// a `Project` passes it only as a direct column item. +fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { + for link in above.iter().rev() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { + column = cols + .iter() + .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; + } + } + Some(column) +} + +/// The identity of `schema`'s column `column`; `None` when its +/// `(table, name)` is not unique in `schema`, so a selection cannot name it. +fn column_identity(schema: &Schema, column: ColumnId) -> Option { + let field = schema.fields.get(column)?; + let namesakes = schema + .fields + .iter() + .filter(|f| f.name == field.name && f.table == field.table) + .count(); + (namesakes == 1).then(|| ColumnIdentity { + table: field.table.clone(), + name: field.name.clone(), + }) +} + +/// The interval on the timestamp column that `links` (top down, the +/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: +/// `(anchor − o − W, anchor − o]` for the one range `TimeRange` of width W +/// and the shifts' offsets summing to o, left-open and right-closed like a +/// PromQL range. The anchor is the evaluation time t +/// (`EvaluationTimePlusMs`), or T for `@ T` (`FixedValue(Timestamp)`). +/// `None` when they are not one interval: no or several range `TimeRange`s, +/// an instant one, several `@`s, or an `@ end()` or `@ start()`, which pins +/// its subtree to one end of the request and is never lifted (the front +/// ends put it above the subtree it anchors). +fn time_constraint(links: &[Operator]) -> Option { + let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); + for link in links { + match link { + Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { + if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { + return None; + } + } + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + offset_ms = offset_ms.checked_add(shift.offset_ms)?; + if let Some(anchor) = shift.at { + if at.replace(anchor).is_some() { + return None; + } + } + } + _ => {} + } + } + let width_ms = i64::try_from(width?.as_millis()).ok()?; + let fixed_anchor = match at { + Some(AtModifier::Timestamp(ms)) => Some(ms), + Some(AtModifier::Start | AtModifier::End) => return None, + None => None, + }; + let before_anchor = |ms: i64| -> Option { + Some(match fixed_anchor { + Some(anchor) => { + IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) + } + None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), + }) + }; + Some(Constraint::Interval { + lower: Bound::Excluded(before_anchor(offset_ms.checked_add(width_ms)?)?), + upper: Bound::Included(before_anchor(offset_ms)?), + }) +} + +/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain +/// (through more of them): the chain's coverage with the interval they +/// select on the timestamp column of the chain's input, in every box. +fn of_window_selection( + selection: &Operator>, +) -> Result { + let mut links = vec![selection.clone()]; + let mut below = *selection.children()[0]; + while let Some(child) = selected_from(&below) { + links.push(below.operator()); + below = child; + } + let only_time_selections = links.iter().all(|link| { + matches!( + link, + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + ) + }); + if !only_time_selections + || !matches!( + below.operator(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + { + return Err(CoverageError::NotSummary); + } + let chain = below + .aggregate_coverage() + .ok_or(CoverageError::NotSummary)?; + let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NotSummary)?; + let constraint = time_constraint(&links).ok_or(CoverageError::NotSummary)?; + let selection = chain + .selection + .iter() + .map(|selection_box| { + let mut columns = selection_box.columns.clone(); + let combined = combined_with(columns.get(&time_column), constraint.clone()) + .ok_or(CoverageError::NotSummary)?; + columns.insert(time_column.clone(), combined); + Ok(SelectionBox { columns }) + }) + .collect::>()?; + Ok(SummaryCoverage { + definition: chain.definition.clone(), + selection, + }) +} + +/// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of +/// equalities on one column, with non-null literals. +fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { + let literal = |e: &ScalarExpr| match e { + ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), + _ => None, + }; + match expr { + ScalarExpr::Compare { + left, op, right, .. + } => { + let (column, value, op) = match (left.as_ref(), right.as_ref()) { + (ScalarExpr::Column(c), r) => (*c, literal(r)?, op.clone()), + (l, ScalarExpr::Column(c)) => (*c, literal(l)?, flip(op)?), + _ => return None, + }; + let constraint = match op { + CompareOpKind::Eq => Constraint::In(vec![value]), + CompareOpKind::Ne => Constraint::NotIn(vec![value]), + CompareOpKind::Lt => interval(Bound::Unbounded, Bound::Excluded(value)), + CompareOpKind::Le => interval(Bound::Unbounded, Bound::Included(value)), + CompareOpKind::Gt => interval(Bound::Excluded(value), Bound::Unbounded), + CompareOpKind::Ge => interval(Bound::Included(value), Bound::Unbounded), + _ => return None, + }; + Some((column, constraint)) + } + ScalarExpr::InList { + expr, + list, + negated, + } => { + let ScalarExpr::Column(column) = expr.as_ref() else { + return None; + }; + let values = list.iter().map(literal).collect::>>()?; + let values = dedup(values); + Some(( + *column, + if *negated { + Constraint::NotIn(values) + } else { + Constraint::In(values) + }, + )) + } + ScalarExpr::BoolOr(disjuncts) => { + let mut column = None; + let mut values = Vec::new(); + for disjunct in disjuncts { + let (c, Constraint::In(v)) = constraint_of(disjunct)? else { + return None; + }; + if column.replace(c).is_some_and(|prev| prev != c) { + return None; + } + values.extend(v); + } + Some((column?, Constraint::In(dedup(values)))) + } + _ => None, + } +} + +fn flip(op: &CompareOpKind) -> Option { + Some(match op { + CompareOpKind::Eq => CompareOpKind::Eq, + CompareOpKind::Ne => CompareOpKind::Ne, + CompareOpKind::Lt => CompareOpKind::Gt, + CompareOpKind::Le => CompareOpKind::Ge, + CompareOpKind::Gt => CompareOpKind::Lt, + CompareOpKind::Ge => CompareOpKind::Le, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::ir::scalar::ExprSemantics; + + use IntervalConstraintLimit::FixedValue; + + /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. + #[test] + fn a_value_comparison_is_an_interval_between_fixed_values() { + let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + }; + assert_eq!( + constraint_of(&price_gt_10), + Some(( + 0, + Constraint::Interval { + lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), + upper: Bound::Unbounded, + } + )) + ); + } +} diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 1a2464a44..86f97e73d 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -8,9 +8,8 @@ use crate::ir::scalar::expr_ir::ColumnRef; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; use crate::ir::ASAPOp; use crate::ir::NonASAPOp; -use crate::stages::{ - CandidateExpression, LogicalProperties, MemoGroupAsInput, OriginOfCandidateForMemoGroup, -}; +use crate::stages::logical::MemoGroupAsInput; +use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate}; /// `Scan metric[1m]`: `[ts, value]`. diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 9c1a8f9fe..280066481 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -1,6 +1,7 @@ //! The logical side: Stage 0's and Stage 1's memo, by scope (memo → group //! → candidate → candidate input → edge → Stage 1's output). +mod aggregate_coverage; mod candidate; mod column_properties; mod edge; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index fa095e9bc..4ad565d9f 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -12,7 +12,6 @@ mod per_computation_logical_memo_candidates; mod physical; mod pruning; -pub(crate) use logical::MemoGroupAsInput; pub use logical::{ queries_rooted_at, root_groups, CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, From f458196bac382abeae1ef485dc94ed949043750e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:24:15 +0000 Subject: [PATCH 04/12] refactor(types, stage0): the memo holds the data workload's source facts; groups record the sources they scan (#1207 PR 2 review) - Memo::new(data) keeps the data workload's facts per source, fixed at construction; a Scan group's column signs read them, a histogram's or summary's series through its family (series_facts, now shared with DataWorkload::series_facts) and a counter's value non-negative. Memo::declare_column_signs and Stage 0's declared_column_signs are deleted. - GroupProperties::scanned_sources replaces reads_only_time_series, with scans_only_time_series(); OperatorInput::reads_only_time_series and its impls are deleted. One rule, scanned_sources in ir/operator/input.rs, derives it: stored per memo group, applied down a node tree by the top-k schema. Pass 1 reads the aggregate group's property. Source derives Ord. --- crates/stage0-memo/src/lib.rs | 47 ++++----------- .../src/computation_targets.rs | 17 +++--- crates/types/src/ir/operator/arguments.rs | 2 +- crates/types/src/ir/operator/input.rs | 35 ++++------- crates/types/src/ir/schema/topk.rs | 17 +++++- .../src/stages/logical/column_properties.rs | 27 ++++++--- crates/types/src/stages/logical/group.rs | 33 +++++++---- crates/types/src/stages/logical/memo/mod.rs | 29 ++++----- crates/types/src/workload/data.rs | 50 +++++++++------- crates/types/src/workload/mod.rs | 1 + crates/types/tests/memo_group_properties.rs | 59 ++++++++++++++----- 11 files changed, 177 insertions(+), 140 deletions(-) diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index 2e120c3dc..cbafe14fd 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -8,22 +8,21 @@ //! where the one expression equality is; no other stage crate depends on //! it, and the planner facade calls it first. -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::rc::Rc; use asap_types::accuracy::AccuracyTarget; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, SchemaDerivationError}; -use asap_types::stages::{Memo, MemoGroupId, MemoQuery, ValueSign}; -use asap_types::workload::bounds::{series_read, source_name, values_per_result}; -use asap_types::workload::{DataWorkload, PrometheusMetricType}; +use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; +use asap_types::stages::{Memo, MemoGroupId, MemoQuery}; +use asap_types::workload::bounds::{series_read, values_per_result}; +use asap_types::workload::DataWorkload; /// Stage 0: the memo groups of the workload's query roots, query `i` /// being `roots[i]` with accuracy target `accuracy_targets[i]` (`None`: /// none), every group without candidates. Each query is added with its /// $n_q$, the number of values one evaluation returns, which `data` bounds /// ([`values_per_result`]); a `Scan`'s column signs are the ones `data` -/// declares ([`declared_column_signs`]), declared to the memo as the walk -/// reaches the source's first `Scan`. Fails with the first node that +/// declares, which the memo holds (`Memo::new`). Fails with the first node that /// breaks the node rules over its children's groups /// ([`Memo::find_or_add_group`]); the facade validates the roots first, so /// that is a frontend bug. @@ -37,11 +36,11 @@ pub fn memo_of_queries( accuracy_targets.len(), "one accuracy target per query" ); - let mut memo = Memo::default(); + let mut memo = Memo::new(data); let mut group_of_node = GroupOfNode::new(); for root in roots { for node in root.operator_refs() { - group_of(&mut memo, &mut group_of_node, node, data)?; + group_of(&mut memo, &mut group_of_node, node)?; } } let group = |node: &Rc| group_of_node[&Rc::as_ptr(node)]; @@ -64,23 +63,6 @@ pub fn memo_of_queries( Ok(memo) } -/// The sign `data` declares for each column of `source`, by column name -/// ([`SourceFacts::column_signs`](asap_types::workload::SourceFacts::column_signs), -/// a histogram's or summary's for its series, [`DataWorkload::series_facts`]); -/// a declared counter's `value` is non-negative without a declaration. -fn declared_column_signs(source: &str, data: &DataWorkload) -> BTreeMap { - let Some((facts, series_type)) = data.series_facts(source) else { - return BTreeMap::new(); - }; - let mut signs = facts.column_signs.clone(); - if series_type == Some(PrometheusMetricType::Counter) { - signs - .entry("value".to_owned()) - .or_insert(ValueSign::NonNegative); - } - signs -} - /// Each node of the queries' DAGs already visited, by address, to its /// group: a node reached through several paths is visited once. The roots /// outlive the map, so no address is reused. @@ -88,23 +70,16 @@ type GroupOfNode = HashMap<*const OperatorNode, MemoGroupId>; /// The group of `node`: its operator over its children's groups (each /// found or added first), found or added by [`Memo::find_or_add_group`]. -/// A `Scan`'s source has its column signs declared to `memo` first -/// ([`declared_column_signs`]). fn group_of( memo: &mut Memo, group_of_node: &mut GroupOfNode, node: &Rc, - data: &DataWorkload, ) -> Result { if let Some(&group) = group_of_node.get(&Rc::as_ptr(node)) { return Ok(group); } for child in node.children() { - group_of(memo, group_of_node, child, data)?; - } - if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.operator { - let source = source_name(source); - memo.declare_column_signs(source, declared_column_signs(source, data)); + group_of(memo, group_of_node, child)?; } let expression = node .operator @@ -117,6 +92,8 @@ fn group_of( #[cfg(test)] mod tests { use super::*; + use std::collections::BTreeMap; + use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::arguments::{BinaryOpKind, GroupKeys, Reduction, Source}; @@ -124,7 +101,9 @@ mod tests { use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{BinaryOperator, NonASAPOp, Operator, ScalarExpr}; + use asap_types::stages::ValueSign; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + use asap_types::workload::PrometheusMetricType; fn node(op: NonASAPOp) -> Rc { OperatorNode::new_shared(Operator::NonASAP(op)).unwrap() diff --git a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs index 4bf2944bc..1372c4c8b 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs @@ -43,7 +43,6 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator::arguments::Reduction; -use asap_types::ir::operator::input::OperatorInput; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; use asap_types::ir::schema::FieldDataType; @@ -462,7 +461,7 @@ pub fn generate_per_computation_candidates( } } add_whole_expression_targets(&mut targets, &logical_nodes); - prune_topk_heaps_over_one_row_per_item(&mut targets, &logical_nodes); + prune_topk_heaps_over_one_row_per_item(memo, &mut targets, &logical_nodes); add_nested_sum_targets(&mut targets, &logical_nodes); close_absorptions(&mut targets); Ok(PerComputationCandidates { @@ -719,13 +718,13 @@ fn add_whole_expression_targets( /// one row per item, so exact is no costlier; a fused whole-expression /// heap ([`add_whole_expression_targets`]) has already moved out of them. fn prune_topk_heaps_over_one_row_per_item( + memo: &Memo, targets: &mut [ComputationTarget], logical_nodes: &[Rc], ) { for target in targets.iter_mut() { - let Some(NonASAPOp::Aggregate { - measures, child, .. - }) = logical_nodes[target.aggregate_group.0 as usize].non_asap() + let Some(NonASAPOp::Aggregate { measures, .. }) = + logical_nodes[target.aggregate_group.0 as usize].non_asap() else { continue; }; @@ -736,8 +735,12 @@ fn prune_topk_heaps_over_one_row_per_item( continue; } // A PromQL top-k ranks an instant vector: one row per series or - // group. (A SQL top-k keeps its heaps.) - if !child.reads_only_time_series() { + // group. (A SQL top-k keeps its heaps.) The aggregate scans what its + // input does. + if !memo.groups()[target.aggregate_group.0 as usize] + .group_properties + .scans_only_time_series() + { continue; } prune_own(&mut target.ways_to_compute, |candidate| { diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index e2a6ef1da..fe0533ffc 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -170,7 +170,7 @@ impl<'de, C: Deserialize<'de>> Deserialize<'de> for GroupKeys { /// The leaf data source of a `Scan`. The schema itself rides on the /// `Scan.schema` field (built by the schema resolver); `Source` carries only the leaf's /// identity. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub enum Source { /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. TimeSeries { metric: String }, diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index caf767173..6aae4811c 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -4,6 +4,7 @@ //! `GroupProperties` (concepts doc `memo-groups-properties-requirements.md`, //! section 3). +use std::collections::BTreeSet; use std::rc::Rc; use super::arguments::Source; @@ -22,24 +23,23 @@ pub trait OperatorInput: Sized + Clone { /// What the input covers when it is an aggregate group in the memo; /// `None` for any other input. fn aggregate_coverage(&self) -> Option<&SummaryCoverage>; - /// Whether every `Scan` below it reads a time series (PromQL rows): - /// [`reads_only_time_series`] of its operator, a memo group's stored - /// property. - fn reads_only_time_series(&self) -> bool; } -/// Whether every `Scan` at or below `operator` reads a time series: a -/// `Scan` of a `Source::TimeSeries`, or any other operator all of whose -/// inputs do. -pub(crate) fn reads_only_time_series(operator: &Operator) -> bool { +/// The sources the `Scan`s at or below `operator` read: a `Scan`'s own, +/// else the union over its inputs, `sources_of(input)` for each. A memo +/// group stores it (`GroupProperties::scanned_sources`); a node tree +/// recomputes it. +pub(crate) fn scanned_sources( + operator: &Operator, + sources_of: impl Fn(&I) -> BTreeSet, +) -> BTreeSet { match operator { - Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { - matches!(source, Source::TimeSeries { .. }) - } + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => BTreeSet::from([source.clone()]), operator => operator .children() .into_iter() - .all(OperatorInput::reads_only_time_series), + .flat_map(sources_of) + .collect(), } } @@ -60,10 +60,6 @@ impl OperatorInput for Rc { fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { None } - - fn reads_only_time_series(&self) -> bool { - reads_only_time_series(&self.operator) - } } /// An input of a rule: a real input `I`, or a build the rule constructs @@ -139,11 +135,4 @@ impl OperatorInput for InputOrTemporaryBuild { Self::TemporaryBuild(_) => None, } } - - fn reads_only_time_series(&self) -> bool { - match self { - Self::Input(input) => input.reads_only_time_series(), - Self::TemporaryBuild(build) => reads_only_time_series(&build.operator), - } - } } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 065a372fb..5a29d61e6 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,5 +1,6 @@ //! Ranked-row output contract shared by logical TopK and its implementations. -use crate::ir::operator::input::OperatorInput; +use crate::ir::operator::arguments::Source; +use crate::ir::operator::input::{scanned_sources, OperatorInput}; use crate::ir::operator::Reduction; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; @@ -55,6 +56,18 @@ pub fn ranked_column(child: &I) -> Result(input: &I) -> bool { + fn sources_below(input: &I) -> std::collections::BTreeSet { + scanned_sources(&input.operator(), sources_below) + } + sources_below(input) + .iter() + .all(|source| matches!(source, Source::TimeSeries { .. })) +} + /// Ordered projection of ranked rows: partition keys, ranked value, then /// item identity. TopK retains rows; partition keys alone are not unique. pub(crate) fn ranked_row_projection( @@ -85,7 +98,7 @@ pub(crate) fn ranked_row_projection( alias: Some(output_name.into()), expr: ScalarExpr::Column(value), }); - let promql = child.reads_only_time_series(); + let promql = scans_only_time_series(child); for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index fe096748a..fa49b8759 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -16,20 +16,21 @@ use crate::ir::schema::{aggregate_output_columns, FieldDataType, OutputColumnSou use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; use crate::workload::bounds::source_name; +use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; use ValueSign::{NonNegative, NonPositive, Unknown}; type Input<'m> = MemoGroupAsInput<'m>; /// The sign of each column of `output`, the output schema of `operator`, -/// from its input groups' `column_sign` and, for a `Scan`, `declared`. +/// from its input groups' `column_sign` and, for a `Scan`, `source_facts`. pub(crate) fn output_column_sign( operator: &Operator>, output: &Schema, - declared: &BTreeMap>, + source_facts: &BTreeMap, ) -> Vec { let signs = match operator { - Operator::NonASAP(op) => non_asap_signs(op, output, declared), + Operator::NonASAP(op) => non_asap_signs(op, output, source_facts), Operator::ASAP(op) => asap_signs(op, output), }; debug_assert_eq!( @@ -48,21 +49,29 @@ fn signs_of(input: &Input<'_>) -> Vec { fn non_asap_signs( op: &NonASAPOp>, output: &Schema, - declared: &BTreeMap>, + source_facts: &BTreeMap, ) -> Vec { use NonASAPOp::*; let unknown = || vec![Unknown; output.fields.len()]; match op { + // The signs its source's facts declare (a histogram's or summary's + // series reads its family's); a counter's `value` is non-negative + // without a declaration. Scan { source, schema, .. } => { - let declared = declared.get(source_name(source)); + let facts = series_facts(source_facts, source_name(source)); schema .fields .iter() - .map(|field| { - declared - .and_then(|columns| columns.get(&field.name)) + .map(|field| match facts { + Some((facts, series_type)) => facts + .column_signs + .get(&field.name) .copied() - .unwrap_or(Unknown) + .or((field.name == "value" + && series_type == Some(PrometheusMetricType::Counter)) + .then_some(NonNegative)) + .unwrap_or(Unknown), + None => Unknown, }) .collect() } diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index e83f95adc..096dc89cc 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -1,9 +1,11 @@ //! Group scope: memo groups, the distinct logical sub-expressions of the //! workload, and the properties every candidate of a group shares. +use std::collections::BTreeSet; + use super::column_properties::output_column_sign; -use crate::ir::operator::arguments::{AtModifier, TimeShift}; -use crate::ir::operator::input::{reads_only_time_series, OperatorInput}; +use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; +use crate::ir::operator::input::{scanned_sources, OperatorInput}; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::schema::Schema; use crate::ir::ASAPOp; @@ -77,10 +79,19 @@ pub struct GroupProperties { /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). pub shareable_when_equal: bool, - /// Whether every `Scan` below reads a time series (PromQL rows): a - /// `Scan` of a `Source::TimeSeries`, or any other operator all of whose - /// input groups do. A PromQL top-k ranks one row per series. - pub reads_only_time_series: bool, + /// The sources the `Scan`s below this group read: a `Scan`'s own, else + /// the union over its input groups. + pub scanned_sources: BTreeSet, +} + +impl GroupProperties { + /// Whether every scanned source is a time series (PromQL rows): a + /// PromQL top-k ranks one row per series. + pub fn scans_only_time_series(&self) -> bool { + self.scanned_sources + .iter() + .all(|source| matches!(source, Source::TimeSeries { .. })) + } } /// The sign of one column's values, a group property derived from the @@ -136,13 +147,15 @@ pub(super) fn derive_group_properties( let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); let (schema, aggregate_coverage) = check_node_rules(&operator)?; Ok(GroupProperties { - column_sign: output_column_sign(&operator, &schema, &memo.declared_column_signs), + column_sign: output_column_sign(&operator, &schema, &memo.source_facts), depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), - reads_only_time_series: reads_only_time_series(&operator), + scanned_sources: scanned_sources(&operator, |input| { + input.group_properties().scanned_sources.clone() + }), }) } @@ -217,8 +230,4 @@ impl OperatorInput for MemoGroupAsInput<'_> { fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { self.group().group_properties.aggregate_coverage.as_ref() } - - fn reads_only_time_series(&self) -> bool { - self.group().group_properties.reads_only_time_series - } } diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index e16fc00e6..fca3624ef 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -13,8 +13,9 @@ use crate::ir::QueryRoot; use crate::ir::SchemaDerivationError; use crate::stages::{ CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalRequirements, MemoGroup, - MemoGroupId, ValueSign, + MemoGroupId, }; +use crate::workload::{DataWorkload, SourceFacts}; use hash_consing::{ expression_hash, same_expression_and_logical_properties, same_serialized_expression, serialized_operator, @@ -34,10 +35,9 @@ pub struct Memo { /// [`expression_hash`] of their expression: every group whose sharing /// is legal ([`GroupProperties::shareable_when_equal`]). groups_by_expression_hash: HashMap)>>, - /// The sign the data workload declares for each source column it - /// declares one for, by source, then column name: a `Scan`'s - /// [`GroupProperties::column_sign`]. - pub(super) declared_column_signs: BTreeMap>, + /// The data workload's facts per source (`DataWorkload::sources`), fixed + /// at construction: a `Scan` group's column signs read them. + pub(super) source_facts: BTreeMap, } /// One query of the memo: its root over groups, its accuracy target and @@ -55,17 +55,14 @@ pub struct MemoQuery { } impl Memo { - /// Declares the sign of each column of `source` its data declares - /// (`column_signs`, by column name), which every `Scan` of `source` - /// added afterwards reads; an undeclared source column's sign is - /// `Unknown`. Stage 0 declares each source before its first `Scan`. - pub fn declare_column_signs( - &mut self, - source: &str, - column_signs: BTreeMap, - ) { - self.declared_column_signs - .insert(source.to_owned(), column_signs); + /// A memo without groups over the sources `data` describes: a `Scan` + /// group's column signs are the ones `data` declares for its source + /// (`GroupProperties::column_sign`). + pub fn new(data: &DataWorkload) -> Self { + Self { + source_facts: data.sources.clone(), + ..Self::default() + } } /// The groups, by id. diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index 124580cd7..aa9e55fc9 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -313,26 +313,7 @@ impl DataWorkload { &self, series: &str, ) -> Option<(&SourceFacts, Option)> { - use PrometheusMetricType::{Counter, Histogram, Summary}; - for (suffix, series_type) in [ - ("_bucket", None), - ("_sum", Some(Counter)), - ("_count", Some(Counter)), - ] { - let Some(family) = series.strip_suffix(suffix) else { - continue; - }; - let Some(facts) = self.sources.get(family) else { - continue; - }; - match (facts.metric_type, series_type) { - (Some(Histogram), None) => return Some((facts, Some(Histogram))), - (Some(Histogram | Summary), Some(counter)) => return Some((facts, Some(counter))), - _ => {} - } - } - let facts = self.sources.get(series)?; - Some((facts, facts.metric_type)) + series_facts(&self.sources, series) } /// Data of unknown arrival whose `metrics` are each sampled every @@ -427,6 +408,35 @@ impl DataWorkload { } } +/// [`DataWorkload::series_facts`] over `sources`, a data workload's +/// [`DataWorkload::sources`]: the facts of the source a PromQL selector over +/// `series` reads, and that series' own type. +pub(crate) fn series_facts<'a>( + sources: &'a BTreeMap, + series: &str, +) -> Option<(&'a SourceFacts, Option)> { + use PrometheusMetricType::{Counter, Histogram, Summary}; + for (suffix, series_type) in [ + ("_bucket", None), + ("_sum", Some(Counter)), + ("_count", Some(Counter)), + ] { + let Some(family) = series.strip_suffix(suffix) else { + continue; + }; + let Some(facts) = sources.get(family) else { + continue; + }; + match (facts.metric_type, series_type) { + (Some(Histogram), None) => return Some((facts, Some(Histogram))), + (Some(Histogram | Summary), Some(counter)) => return Some((facts, Some(counter))), + _ => {} + } + } + let facts = sources.get(series)?; + Some((facts, facts.metric_type)) +} + fn validate_optional_rate(rate: Option) -> Result<(), WorkloadError> { rate.map(validate_rate).transpose().map(|_| ()) } diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index 5b4f20270..2af8e5966 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -8,6 +8,7 @@ pub mod parsed_workload; mod planning; mod query; +pub(crate) use data::series_facts; pub use data::{ DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, StreamNorms, diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 229ebfff7..a14bd6e14 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -7,7 +7,7 @@ //! requests_total [ts, value, host] a counter: value non-negative //! gauge [ts, value, host] nothing declared //! ``` -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use asap_types::ir::operator::arguments::{AtModifier, GroupKeys, JoinKind, Reduction, Source}; use asap_types::ir::operator::AggIntent; @@ -21,6 +21,7 @@ use asap_types::ir::{ }; use asap_types::stages::{Memo, MemoGroupId, ValueSign}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; +use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; use ValueSign::{NonNegative, NonPositive, Unknown}; @@ -30,13 +31,27 @@ const HOST: usize = 2; const MINUTE_MS: i64 = 60_000; /// The memo of every example: `latency.value` declared non-negative, and -/// `requests_total.value` as Stage 0 declares a counter's. +/// `requests_total` a declared counter, whose `value` is non-negative. fn memo() -> Memo { - let mut memo = Memo::default(); - for source in ["latency", "requests_total"] { - memo.declare_column_signs(source, BTreeMap::from([("value".into(), NonNegative)])); - } - memo + Memo::new(&DataWorkload { + sources: BTreeMap::from([ + ( + "latency".to_string(), + SourceFacts { + column_signs: BTreeMap::from([("value".to_string(), NonNegative)]), + ..SourceFacts::default() + }, + ), + ( + "requests_total".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + ..SourceFacts::default() + }, + ), + ]), + ..DataWorkload::default() + }) } fn add(memo: &mut Memo, expression: Operator) -> MemoGroupId { @@ -408,12 +423,14 @@ fn a_result_depends_on_the_evaluation_time_under_a_time_range_or_now() { } } -/// `reads_only_time_series`, derived once per group: a PromQL `Scan` of a -/// time series reads one, and so does a `TimeRange` over it; a SQL table -/// `Scan` does not, nor a `Join` reading it (a SQL join reads relations -/// only, so a time series cannot be one of its inputs). +/// `scanned_sources`, derived once per group: a `Scan` its source, any other +/// group the union over its input groups. A PromQL `Scan` of a time series +/// and a `TimeRange` over it scan only time series +/// (`scans_only_time_series`); a SQL table `Scan` does not, nor a `Join` +/// reading two (a SQL join reads relations only, so a time series cannot be +/// one of its inputs). #[test] -fn a_group_reads_only_time_series_when_every_scan_below_reads_one() { +fn a_group_scans_the_sources_of_the_scans_below_it() { let mut memo = memo(); let series = add(&mut memo, series("latency")); let window = add(&mut memo, range(MINUTE_MS, series)); @@ -440,11 +457,21 @@ fn a_group_reads_only_time_series_when_every_scan_below_reads_one() { right: hosts, }), ); - let reads = |group: MemoGroupId| { - memo.groups()[group.0 as usize] - .group_properties - .reads_only_time_series + let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; + let table_source = |name: &str| Source::Table { + table_ref: name.into(), }; + assert_eq!( + properties(join).scanned_sources, + BTreeSet::from([table_source("hosts"), table_source("requests")]) + ); + assert_eq!( + properties(window).scanned_sources, + BTreeSet::from([Source::TimeSeries { + metric: "latency".into() + }]) + ); + let reads = |group: MemoGroupId| properties(group).scans_only_time_series(); assert!(reads(series)); assert!(reads(window)); assert!(!reads(requests)); From 0fef1000175ba70a6fb176740c4ffa16fe8d3eee Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:26:37 +0000 Subject: [PATCH 05/12] refactor(types): name the memo's expression check by what it checks (#1207 PR 2 review) check_node_rules -> validate_expression_over_input_groups (an expression over input groups is valid when its output schema derives, its inputs pass validation, and a merge reads only states it may merge). Its one other caller, check_candidate_expression, is inlined into Memo::add_candidate. Docs, the CandidateError::Schema message and a Stage 0 test name say what is checked instead of "node rules". --- crates/stage0-memo/src/lib.rs | 11 ++++---- crates/types/src/stages/logical/candidate.rs | 7 ++--- crates/types/src/stages/logical/group.rs | 26 ++++++------------- crates/types/src/stages/logical/memo/mod.rs | 27 ++++++++++++++------ 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index cbafe14fd..fdb287af1 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -22,8 +22,8 @@ use asap_types::workload::DataWorkload; /// none), every group without candidates. Each query is added with its /// $n_q$, the number of values one evaluation returns, which `data` bounds /// ([`values_per_result`]); a `Scan`'s column signs are the ones `data` -/// declares, which the memo holds (`Memo::new`). Fails with the first node that -/// breaks the node rules over its children's groups +/// declares, which the memo holds (`Memo::new`). Fails with the first node +/// whose expression is not valid over its children's groups /// ([`Memo::find_or_add_group`]); the facade validates the roots first, so /// that is a frontend bug. pub fn memo_of_queries( @@ -496,10 +496,11 @@ mod tests { assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); } - /// A frontend node that breaks the node rules (a readout of raw rows - /// instead of summary state) makes Stage 0 return the error. + /// A frontend node whose expression is not valid over its children's + /// groups (a readout of raw rows instead of summary state) makes Stage 0 + /// return the error. #[test] - fn memo_of_queries_returns_the_error_of_a_node_that_breaks_the_node_rules() { + fn memo_of_queries_returns_the_error_of_a_node_invalid_over_its_input_groups() { let readout_of_rows = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: scan(), query: SketchStatistic::Quantile { q: 0.5 }, diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs index f1f89ab30..06919793b 100644 --- a/crates/types/src/stages/logical/candidate.rs +++ b/crates/types/src/stages/logical/candidate.rs @@ -65,9 +65,10 @@ pub struct CandidateExpression { /// A candidate [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects; the memo is unchanged. #[derive(Debug, thiserror::Error)] pub enum CandidateError { - /// The candidate's expression fails the node rules (schema derivation, - /// input validation, merge check): a bug in the rule that built it. - #[error("the candidate's expression breaks the node rules: {0}")] + /// The candidate's expression is not valid over its input groups (its + /// output schema does not derive, an input fails validation, or a merge + /// reads states it may not merge): a bug in the rule that built it. + #[error("the candidate's expression is not valid over its input groups: {0}")] Schema(SchemaDerivationError), /// Its output schema or output kind differs from the group's: it does /// not compute the group's result (a bug in the rule that built it). diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 096dc89cc..724089970 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -106,11 +106,11 @@ pub enum ValueSign { Unknown, } -/// The node rules every group and candidate expression must pass, over -/// its input groups ([`MemoGroupAsInput`]), never a node tree: schema -/// derivation, input validation and, for a merge, the coverage check. +/// Whether `operator` is a valid expression over its input groups +/// ([`MemoGroupAsInput`]), never a node tree: its output schema derives, its +/// inputs pass validation, and a merge reads only states it may merge. /// Returns the output schema and the aggregate coverage. -fn check_node_rules( +pub(super) fn validate_expression_over_input_groups( operator: &Operator>, ) -> Result<(Schema, Option), SchemaDerivationError> { let schema = operator.output_schema()?; @@ -126,26 +126,16 @@ fn check_node_rules( Ok((schema, aggregate_coverage)) } -/// The output schema and kind of a candidate computing `expression` over -/// its input groups, once it passes the node rules ([`check_node_rules`]). -pub(super) fn check_candidate_expression( - memo: &Memo, - expression: &Operator, -) -> Result<(Schema, OperatorResultKind), SchemaDerivationError> { - let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); - let (schema, _) = check_node_rules(&operator)?; - Ok((schema, operator.output_kind())) -} - /// The properties of a group computing `expression` over its input -/// groups, once it passes the node rules ([`check_node_rules`]), reading -/// each input group's `group_properties` and `expression`. +/// groups, once it is valid over them +/// ([`validate_expression_over_input_groups`]), reading each input group's +/// `group_properties` and `expression`. pub(super) fn derive_group_properties( memo: &Memo, expression: &Operator, ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); - let (schema, aggregate_coverage) = check_node_rules(&operator)?; + let (schema, aggregate_coverage) = validate_expression_over_input_groups(&operator)?; Ok(GroupProperties { column_sign: output_column_sign(&operator, &schema, &memo.source_facts), depends_on_evaluation_time: depends_on_evaluation_time(&operator), diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index fca3624ef..8de0b4cae 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -6,7 +6,9 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; mod hash_consing; -use super::group::{check_candidate_expression, derive_group_properties}; +use super::group::{ + derive_group_properties, validate_expression_over_input_groups, MemoGroupAsInput, +}; use crate::accuracy::AccuracyTarget; use crate::ir::Operator; use crate::ir::QueryRoot; @@ -88,8 +90,9 @@ impl Memo { /// group. A group whose copies may return different rows (an unordered /// `LIMIT`, [`GroupProperties::shareable_when_equal`]) is never returned /// for another expression, so such an expression always gets a new - /// group. Fails, leaving the memo unchanged, when the expression breaks - /// the node rules (schema derivation, input validation, merge check). + /// group. Fails, leaving the memo unchanged, when the expression is not + /// valid over its input groups (its output schema does not derive, an + /// input fails validation, or a merge reads states it may not merge). pub fn find_or_add_group( &mut self, expression: Operator, @@ -97,7 +100,7 @@ impl Memo { let serialized = serialized_operator(&expression); let hash = expression_hash(&expression, serialized.as_deref()); // Only shareable groups are listed by hash, and an equal expression - // is as shareable as the group it equals, which passed the node rules. + // is as shareable as the group it equals, which was validated. if let Some((group, _)) = self .groups_by_expression_hash .get(&hash) @@ -144,7 +147,8 @@ impl Memo { /// group and every group below it. /// /// Fails, leaving the memo unchanged, when the candidate's expression - /// breaks the node rules a group's does ([`check_candidate_expression`]), + /// is not valid over its input groups + /// ([`validate_expression_over_input_groups`]), /// or when its output schema or kind differs from the group's: then it /// does not compute the group's result. pub fn add_candidate( @@ -162,9 +166,16 @@ impl Memo { index_in_group, }); } - let (schema, result_kind) = - check_candidate_expression(self, &candidate.expression.operator) - .map_err(CandidateError::Schema)?; + let operator = candidate + .expression + .operator + .map_children(|&group_id| MemoGroupAsInput { + memo: self, + group_id, + }); + let (schema, _) = + validate_expression_over_input_groups(&operator).map_err(CandidateError::Schema)?; + let result_kind = operator.output_kind(); let group_properties = &self.groups[group.0 as usize].group_properties; if schema != group_properties.schema || result_kind != group_properties.result_kind { return Err(CandidateError::OutputDiffersFromGroup { group }); From 1ae8acf91c8481dd8d50f582116815ef4dbe0152 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:48:15 +0000 Subject: [PATCH 06/12] refactor(types, stage0): second review of the memo: aggregate coverage names, one layout pass, one equality (#1207 PR 2 review) - D1: OperatorInput::scanned_sources (a memo group's stored set, recomputed for node trees by sources_scanned_by); one scans_only_time_series over a set, read by the top-k schema and Pass 1; topk.rs's own walk is deleted. - D2: SummaryCoverage -> AggregateCoverage, SummaryStateDefinition -> AggregateDefinition, CoverageError::NotSummary -> NoCoverage; ir/properties/aggregate_coverage.rs; tests/memo_aggregate_coverage.rs. - D4: constraint_of and flip beside Constraint in ir/properties; dedup and interval private again. - M1: one expression equality (same_serialized_expression). - M2: OutputColumnSource::Measure { index, input_column }; the signs read the measure's input column from the layout. - M3: summary_agg_layout replaces summary_agg_schema and summary_agg_columns. - M4: validate_expression_over_input_groups returns the schema and derives coverage only to check a merge. - M5, M6: stale docs and doc links. - M7: column_sign -> column_signs; CandidateError::Schema -> NotValidOverInputGroups; depends_on_it inlined. - M8: MemoGroupAsInput, its fields, group_properties, output_column_sign and derive_from_inputs are pub(super). - M9: Stage 0's hash-consing tests move to stages/logical/memo/tests.rs, built through find_or_add_group. - M10: Memo has no Default; Memo::new(&DataWorkload::default()) in tests. --- crates/planner/src/lib.rs | 2 +- crates/stage0-memo/src/lib.rs | 239 ++--------------- .../src/computation_targets.rs | 10 +- crates/types/src/ir/operator/asap.rs | 55 ++-- crates/types/src/ir/operator/input.rs | 42 ++- ...mary_coverage.rs => aggregate_coverage.rs} | 126 ++++++++- crates/types/src/ir/properties/mod.rs | 5 +- .../types/src/ir/schema/aggregate_schema.rs | 112 ++++++-- crates/types/src/ir/schema/error.rs | 2 +- crates/types/src/ir/schema/topk.rs | 19 +- crates/types/src/plan/materialized_view.rs | 14 +- crates/types/src/plan/physical_asap_dag.rs | 2 +- .../src/stages/logical/aggregate_coverage.rs | 174 +++--------- crates/types/src/stages/logical/candidate.rs | 2 +- .../src/stages/logical/column_properties.rs | 93 +++---- crates/types/src/stages/logical/group.rs | 69 ++--- .../src/stages/logical/memo/hash_consing.rs | 29 +- crates/types/src/stages/logical/memo/mod.rs | 20 +- crates/types/src/stages/logical/memo/tests.rs | 248 +++++++++++++++++- crates/types/src/stages/logical/mod.rs | 2 +- .../src/stages/physical/memo_candidates.rs | 5 +- crates/types/src/stages/pruning.rs | 2 +- crates/types/src/test_support.rs | 6 +- crates/types/src/workload/data.rs | 2 +- crates/types/tests/main.rs | 2 +- ...coverage.rs => memo_aggregate_coverage.rs} | 31 +-- crates/types/tests/memo_fixtures.rs | 2 +- crates/types/tests/memo_group_properties.rs | 26 +- .../architecture/input-output-workflow.md | 2 +- .../proposals/asap-primitive-schema.md | 48 ++-- .../asap-aware-mapping-contracts.md | 2 +- 31 files changed, 723 insertions(+), 670 deletions(-) rename crates/types/src/ir/properties/{summary_coverage.rs => aggregate_coverage.rs} (83%) rename crates/types/tests/{memo_summary_coverage.rs => memo_aggregate_coverage.rs} (96%) diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index dc6fb1350..21e6cf6a5 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -57,7 +57,7 @@ pub use asap_stage3_plan_selection::{ AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, }; pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; -pub use asap_types::ir::properties::summary_coverage::{ +pub use asap_types::ir::properties::aggregate_coverage::{ ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, }; pub use asap_types::ir::properties::ExecutionTiming; diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index fdb287af1..de5a9bed5 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -96,14 +96,13 @@ mod tests { use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::agg_intent::AggIntent; - use asap_types::ir::operator::arguments::{BinaryOpKind, GroupKeys, Reduction, Source}; + use asap_types::ir::operator::arguments::{BinaryOpKind, Reduction, Source}; use asap_types::ir::operator::asap::ASAPOp; use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{BinaryOperator, NonASAPOp, Operator, ScalarExpr}; use asap_types::stages::ValueSign; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; - use asap_types::workload::PrometheusMetricType; fn node(op: NonASAPOp) -> Rc { OperatorNode::new_shared(Operator::NonASAP(op)).unwrap() @@ -172,12 +171,6 @@ mod tests { } } - /// Whether queries 0 and 1 of `[a, b]` are one group. - fn one_group(a: Rc, b: Rc) -> bool { - let memo = memo_of(vec![a, b]); - root(&memo, 0) == root(&memo, 1) - } - /// Two queries sharing a sub-expression read one group for it, and /// every group lists the queries that reach it and the groups that /// read it. @@ -238,78 +231,6 @@ mod tests { assert_eq!(memo.groups().len(), 3); } - /// Aggregates that differ in a column, a quantile or a filter are - /// different groups. - #[test] - fn aggregates_that_differ_in_an_argument_are_different_groups() { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::Predicate; - assert!(!one_group( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(3), 0.5) - )); - assert!(!one_group( - quantile_agg(vec![], Some(2), 0.5), - quantile_agg(vec![], Some(2), 0.9) - )); - let filtered = node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Quantile { - col: Some(2), - q: 0.5, - }], - output_names: vec![], - filters: vec![Some(Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(3)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: asap_types::ir::ExprSemantics::Sql, - }))], - having: None, - child: scan(), - }); - assert!(!one_group(quantile_agg(vec![1], Some(2), 0.5), filtered)); - } - - /// Equal nodes without a unique key are one group when they return the - /// same rows each time: a keyless scan, a global aggregate, a `without` - /// aggregate. - #[test] - fn equal_keyless_nodes_that_return_the_same_rows_are_one_group() { - let table_scan = || { - node(NonASAPOp::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - }) - }; - assert!(one_group(table_scan(), table_scan())); - assert!(one_group( - quantile_agg(vec![], Some(2), 0.9), - quantile_agg(vec![], Some(2), 0.9) - )); - let without = || { - node(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }) - }; - assert!(one_group(without(), without())); - } - /// Queries that differ only in their accuracy target are one group: the /// target is the query's ([`MemoQuery::accuracy_target`]), not an /// argument of its operators. @@ -401,101 +322,6 @@ mod tests { assert_eq!(memo.groups().len(), 4); } - /// `DISTINCT` on every column returns the same rows each time and is one - /// group; a `Dedup` on some columns keeps an unspecified row per key. - #[test] - fn dedup_is_one_group_only_on_every_column() { - let dedup = |cols: Vec| { - node(NonASAPOp::Dedup { - cols, - child: scan(), - }) - }; - assert!(one_group(dedup(vec![]), dedup(vec![]))); - assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); - } - - fn unordered_limit(child: Rc) -> Rc { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child, - }) - } - - /// Two unordered `LIMIT 10`s may return different rows: two groups, and - /// so are equal parents over them. - #[test] - fn unordered_limits_and_their_parents_are_different_groups() { - assert!(!one_group(unordered_limit(scan()), unordered_limit(scan()))); - let distinct_over_limit = || { - node(NonASAPOp::Dedup { - cols: vec![], - child: unordered_limit(scan()), - }) - }; - assert!(!one_group(distinct_over_limit(), distinct_over_limit())); - } - - /// A `LIMIT` over a `Sort` with the same partitions keeps the first rows - /// of a fixed order (one group); over a `Sort` partitioned differently, - /// which rows come first depends on input order (two groups). - #[test] - fn limits_over_a_sort_with_their_partitions_are_one_group() { - let limit = |sort_partitions: GroupKeys| { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: node(NonASAPOp::Sort { - keys: vec![asap_types::ir::scalar::SortKey { - expr: ScalarExpr::Column(2), - ascending: false, - nulls_first: false, - }], - partition_by: sort_partitions, - child: scan(), - }), - }) - }; - assert!(one_group( - limit(GroupKeys::default()), - limit(GroupKeys::default()) - )); - assert!(!one_group( - limit(GroupKeys::by(vec![1])), - limit(GroupKeys::by(vec![1])) - )); - } - - /// A projection emitting the literal `value`. - fn literal(value: f64) -> Rc { - node(NonASAPOp::Project { - cols: vec![asap_types::ir::ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::literal_f64(value), - }], - qualifier: None, - child: scan(), - }) - } - - /// Signed zeros and different non-finite literals are different groups; - /// JSON's `null` for non-finite floats never decides equality. - #[test] - fn signed_zero_and_non_finite_literals_are_different_groups() { - for (a, b) in [ - (0.0, -0.0), - (-0.0, 0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); - } - assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); - } - /// A frontend node whose expression is not valid over its children's /// groups (a readout of raw rows instead of summary state) makes Stage 0 /// return the error. @@ -556,59 +382,28 @@ mod tests { ); } - /// A `Scan`'s column signs are the data workload's declared ones: `m` - /// declares `latency` non-negative, the counter `requests_total`'s - /// `value` is non-negative without a declaration, and every other - /// column is `Unknown`. + /// `memo_of_queries` passes the data workload's facts to the memo: `m` + /// declares `latency` non-negative, which its `Scan` group's signs read. #[test] fn a_scan_reads_the_declared_column_signs() { - let counter = node(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "requests_total".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - }); let data = DataWorkload { - sources: BTreeMap::from([ - ( - "m".to_owned(), - asap_types::workload::SourceFacts { - column_signs: BTreeMap::from([( - "latency".to_owned(), - ValueSign::NonNegative, - )]), - ..Default::default() - }, - ), - ( - "requests_total".to_owned(), - asap_types::workload::SourceFacts { - metric_type: Some(PrometheusMetricType::Counter), - ..Default::default() - }, - ), - ]), + sources: BTreeMap::from([( + "m".to_owned(), + asap_types::workload::SourceFacts { + column_signs: BTreeMap::from([("latency".to_owned(), ValueSign::NonNegative)]), + ..Default::default() + }, + )]), ..Default::default() }; - let roots = [QueryRoot::Operator(scan()), QueryRoot::Operator(counter)]; - let memo = memo_of_queries(&roots, &[None, None], &data).unwrap(); + let memo = memo_of_queries(&[QueryRoot::Operator(scan())], &[None], &data).unwrap(); use ValueSign::{NonNegative, Unknown}; - let signs = |query| { - &memo.groups()[root(&memo, query).0 as usize] - .group_properties - .column_sign - }; // [ts, service, value, latency] - assert_eq!(signs(0), &[Unknown, Unknown, Unknown, NonNegative]); - // [ts, value] - assert_eq!(signs(1), &[Unknown, NonNegative]); + assert_eq!( + memo.groups()[root(&memo, 0).0 as usize] + .group_properties + .column_signs, + [Unknown, Unknown, Unknown, NonNegative] + ); } } diff --git a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs index 1372c4c8b..1cf6f0258 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs @@ -43,6 +43,7 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator::arguments::Reduction; +use asap_types::ir::operator::input::scans_only_time_series; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; use asap_types::ir::schema::FieldDataType; @@ -737,10 +738,11 @@ fn prune_topk_heaps_over_one_row_per_item( // A PromQL top-k ranks an instant vector: one row per series or // group. (A SQL top-k keeps its heaps.) The aggregate scans what its // input does. - if !memo.groups()[target.aggregate_group.0 as usize] - .group_properties - .scans_only_time_series() - { + if !scans_only_time_series( + &memo.groups()[target.aggregate_group.0 as usize] + .group_properties + .scanned_sources, + ) { continue; } prune_own(&mut target.ways_to_compute, |candidate| { diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index e63fe3f7a..f24a426b6 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -12,7 +12,8 @@ use super::non_asap::NonASAPOp; use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::arguments::Reduction; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; -use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; +use crate::ir::schema::aggregate_schema::aggregate_output; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, OutputColumnSource, Schema}; use crate::ir::SchemaDerivationError; use crate::summary_type::SummaryTypeCapability; use crate::summary_type::{SketchStatistic, SummaryUpdate}; @@ -285,7 +286,7 @@ impl ASAPOp { summary_type: family, reduction, .. - } => summary_agg_schema(child.schema(), family, reduction)?, + } => summary_agg_layout(child.schema(), family, reduction)?.0, SummaryTumblingWindowChain { child, summary_type: family, @@ -427,7 +428,7 @@ impl ASAPOp { reduction, .. }), - ) => summary_agg_schema(child.schema(), &summary_type, &reduction)?, + ) => summary_agg_layout(child.schema(), &summary_type, &reduction)?.0, (None, _) => input.clone(), } } @@ -721,7 +722,7 @@ fn rolled_up_build( }) .collect::>>()?; let reduction = Reduction::by(mapped); - let schema = summary_agg_schema(child.schema(), &summary_type, &reduction).ok()?; + let (schema, _) = summary_agg_layout(child.schema(), &summary_type, &reduction).ok()?; Some(( ASAPOp::SummaryAgg { child, @@ -735,43 +736,31 @@ fn rolled_up_build( } /// The schema of a `SummaryAgg` (or `SummaryTumblingWindowChain`) of -/// `summary_type` by `reduction` over rows of schema `child`: the reduction's -/// output columns, its value column replaced by the state. -pub(crate) fn summary_agg_schema( +/// `summary_type` by `reduction` over rows of schema `child`, and where each +/// of its columns comes from: the reduction's output columns, its one +/// measure's value column replaced by the state +/// (`OutputColumnSource::Measure`). +pub(crate) fn summary_agg_layout( child: &Schema, summary_type: &FieldDataType, reduction: &Reduction, -) -> Result { - let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( +) -> Result<(Schema, Vec), SchemaDerivationError> { + let (mut schema, sources) = aggregate_output( child, reduction, &[crate::ir::operator::AggIntent::Sum { col: None }], &[], )?; - let index = match reduction { - Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( - &crate::ir::scalar::ColumnRef::SampleValue, - &schema, - ) - .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, - Reduction::Reduce(_) => schema.fields.len() - 1, - }; + let index = sources + .iter() + .position(|source| matches!(source, OutputColumnSource::Measure { .. })) + .ok_or_else(|| { + SchemaDerivationError::InvalidScalarSignature( + "a summary state has no value column".into(), + ) + })?; schema.fields[index] = Field::new("state", summary_type.clone(), false); - Ok(schema) -} - -/// Where each column of [`summary_agg_schema`]'s output comes from: the -/// aggregate layout of `reduction` with the state in the one measure's place -/// (`OutputColumnSource::Measure(0)`). -pub(crate) fn summary_agg_columns( - child: &Schema, - reduction: &Reduction, -) -> Result, SchemaDerivationError> { - crate::ir::schema::aggregate_output_columns( - child, - reduction, - &[crate::ir::operator::AggIntent::Sum { col: None }], - ) + Ok((schema, sources)) } /// The schema of a `SummaryTumblingWindowChain` of `summary_type` by `reduction` @@ -785,7 +774,7 @@ fn chain_schema( summary_type: &FieldDataType, reduction: &Reduction, ) -> Result { - let mut schema = summary_agg_schema(child, summary_type, reduction)?; + let (mut schema, _) = summary_agg_layout(child, summary_type, reduction)?; let time = child .time_index .and_then(|index| child.fields.get(index)) diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index 6aae4811c..a489a1047 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -10,7 +10,7 @@ use std::rc::Rc; use super::arguments::Source; use super::node::{Operator, OperatorNode, OperatorResultKind}; use super::non_asap::NonASAPOp; -use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::properties::aggregate_coverage::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; @@ -22,27 +22,34 @@ pub trait OperatorInput: Sized + Clone { fn operator(&self) -> Operator; /// What the input covers when it is an aggregate group in the memo; /// `None` for any other input. - fn aggregate_coverage(&self) -> Option<&SummaryCoverage>; + fn aggregate_coverage(&self) -> Option<&AggregateCoverage>; + /// The sources the `Scan`s at or below the input read + /// (`sources_scanned_by` of its operator): a memo group's stored + /// `GroupProperties::scanned_sources`, recomputed for a node tree. + fn scanned_sources(&self) -> BTreeSet; } /// The sources the `Scan`s at or below `operator` read: a `Scan`'s own, -/// else the union over its inputs, `sources_of(input)` for each. A memo -/// group stores it (`GroupProperties::scanned_sources`); a node tree -/// recomputes it. -pub(crate) fn scanned_sources( - operator: &Operator, - sources_of: impl Fn(&I) -> BTreeSet, -) -> BTreeSet { +/// else the union over its inputs' ([`OperatorInput::scanned_sources`]). +pub(crate) fn sources_scanned_by(operator: &Operator) -> BTreeSet { match operator { Operator::NonASAP(NonASAPOp::Scan { source, .. }) => BTreeSet::from([source.clone()]), operator => operator .children() .into_iter() - .flat_map(sources_of) + .flat_map(OperatorInput::scanned_sources) .collect(), } } +/// Whether every source in `sources` is a time series (PromQL rows): a +/// PromQL top-k ranks one row per series. +pub fn scans_only_time_series(sources: &BTreeSet) -> bool { + sources + .iter() + .all(|source| matches!(source, Source::TimeSeries { .. })) +} + /// A front-end node tree. Node trees carry no summary coverage. impl OperatorInput for Rc { fn schema(&self) -> &Schema { @@ -57,9 +64,13 @@ impl OperatorInput for Rc { self.operator.clone() } - fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { None } + + fn scanned_sources(&self) -> BTreeSet { + sources_scanned_by(&self.operator) + } } /// An input of a rule: a real input `I`, or a build the rule constructs @@ -129,10 +140,17 @@ impl OperatorInput for InputOrTemporaryBuild { } } - fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { match self { Self::Input(input) => input.aggregate_coverage(), Self::TemporaryBuild(_) => None, } } + + fn scanned_sources(&self) -> BTreeSet { + match self { + Self::Input(input) => input.scanned_sources(), + Self::TemporaryBuild(_) => sources_scanned_by(&self.operator()), + } + } } diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/aggregate_coverage.rs similarity index 83% rename from crates/types/src/ir/properties/summary_coverage.rs rename to crates/types/src/ir/properties/aggregate_coverage.rs index 349332a0b..aec4f7815 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/aggregate_coverage.rs @@ -1,5 +1,6 @@ -//! What a summary state covers: `definition` (what it computes) and -//! `selection` (which output rows of that computation it took). Design: +//! Aggregate coverage: what an aggregate (a summary state or an exact +//! `Aggregate`) covers: `definition` (what it computes) and `selection` (which +//! output rows of that computation it took). Design: //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! @@ -13,14 +14,15 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::ir::operator::node::Operator; -use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::ScalarExpr; use crate::ir::schema::ColumnId; use crate::stages::MemoGroupId; use crate::summary_type::MergeInputRule; -/// What a summary state computes, with every lifted selection removed. +/// What an aggregate computes, with every lifted selection removed. #[derive(Debug, Clone, PartialEq)] -pub struct SummaryStateDefinition { +pub struct AggregateDefinition { /// The group the unlifted operators read: the first input under the /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ /// / direct-column Project. `None` when that input is a `Scan` with a @@ -38,15 +40,15 @@ pub struct SummaryStateDefinition { } #[derive(Debug, Clone, PartialEq)] -pub struct SummaryCoverage { - pub(crate) definition: SummaryStateDefinition, +pub struct AggregateCoverage { + pub(crate) definition: AggregateDefinition, /// Union of boxes over the output rows of the unlifted operators' input. pub(crate) selection: Vec, } #[cfg(any(test, feature = "test-support"))] -impl SummaryCoverage { - pub fn definition(&self) -> &SummaryStateDefinition { +impl AggregateCoverage { + pub fn definition(&self) -> &AggregateDefinition { &self.definition } @@ -227,7 +229,7 @@ pub enum CoverageError { /// interval on the timestamp column. A caller bug, or bad input when it /// is a merge's input. #[error("coverage is derived only for aggregates and one time interval over a chain")] - NotSummary, + NoCoverage, /// A `SummaryMerge` without inputs: bad input. #[error("summary merge requires at least one input")] EmptyMerge, @@ -262,7 +264,7 @@ pub enum CoverageError { RegroupedBuild { reason: String }, } -impl SummaryCoverage { +impl AggregateCoverage { /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, /// of its input group and unlifted operators. Two equal definitions /// (the equality a merge requires) of one memo get equal ids, so a @@ -279,15 +281,91 @@ impl SummaryCoverage { } } +/// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of +/// equalities on one column, with non-null literals. +pub(crate) fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { + let literal = |e: &ScalarExpr| match e { + ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), + _ => None, + }; + match expr { + ScalarExpr::Compare { + left, op, right, .. + } => { + let (column, value, op) = match (left.as_ref(), right.as_ref()) { + (ScalarExpr::Column(c), r) => (*c, literal(r)?, op.clone()), + (l, ScalarExpr::Column(c)) => (*c, literal(l)?, flip(op)?), + _ => return None, + }; + let constraint = match op { + CompareOpKind::Eq => Constraint::In(vec![value]), + CompareOpKind::Ne => Constraint::NotIn(vec![value]), + CompareOpKind::Lt => interval(Bound::Unbounded, Bound::Excluded(value)), + CompareOpKind::Le => interval(Bound::Unbounded, Bound::Included(value)), + CompareOpKind::Gt => interval(Bound::Excluded(value), Bound::Unbounded), + CompareOpKind::Ge => interval(Bound::Included(value), Bound::Unbounded), + _ => return None, + }; + Some((column, constraint)) + } + ScalarExpr::InList { + expr, + list, + negated, + } => { + let ScalarExpr::Column(column) = expr.as_ref() else { + return None; + }; + let values = list.iter().map(literal).collect::>>()?; + let values = dedup(values); + Some(( + *column, + if *negated { + Constraint::NotIn(values) + } else { + Constraint::In(values) + }, + )) + } + ScalarExpr::BoolOr(disjuncts) => { + let mut column = None; + let mut values = Vec::new(); + for disjunct in disjuncts { + let (c, Constraint::In(v)) = constraint_of(disjunct)? else { + return None; + }; + if column.replace(c).is_some_and(|prev| prev != c) { + return None; + } + values.extend(v); + } + Some((column?, Constraint::In(dedup(values)))) + } + _ => None, + } +} + +fn flip(op: &CompareOpKind) -> Option { + Some(match op { + CompareOpKind::Eq => CompareOpKind::Eq, + CompareOpKind::Ne => CompareOpKind::Ne, + CompareOpKind::Lt => CompareOpKind::Gt, + CompareOpKind::Le => CompareOpKind::Ge, + CompareOpKind::Gt => CompareOpKind::Lt, + CompareOpKind::Ge => CompareOpKind::Le, + _ => return None, + }) +} + /// An interval between two fixed values. -pub(crate) fn interval(lower: Bound, upper: Bound) -> Constraint { +fn interval(lower: Bound, upper: Bound) -> Constraint { Constraint::Interval { lower: lower.map(IntervalConstraintLimit::FixedValue), upper: upper.map(IntervalConstraintLimit::FixedValue), } } -pub(crate) fn dedup(values: Vec) -> Vec { +fn dedup(values: Vec) -> Vec { let mut out: Vec = Vec::new(); for v in values { if !out.contains(&v) { @@ -586,6 +664,7 @@ fn touching( #[cfg(test)] mod tests { use super::*; + use crate::ir::scalar::ExprSemantics; use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; @@ -635,4 +714,25 @@ mod tests { constraint ); } + + /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. + #[test] + fn a_value_comparison_is_an_interval_between_fixed_values() { + let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + }; + assert_eq!( + constraint_of(&price_gt_10), + Some(( + 0, + Constraint::Interval { + lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), + upper: Bound::Unbounded, + } + )) + ); + } } diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 5eecc5878..b491c013a 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -3,8 +3,9 @@ //! //! - [`execution_data_state`] — when a node's value is produced //! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). -//! - [`summary_coverage`] — what a summary state covers. +//! - [`aggregate_coverage`] — what an aggregate (a summary state or an exact +//! `Aggregate`) covers. +pub mod aggregate_coverage; pub mod execution_data_state; -pub mod summary_coverage; pub use execution_data_state::{DataPrimitive, ExecutionDataState, ExecutionTiming}; diff --git a/crates/types/src/ir/schema/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs index 345cb78cb..240d1863a 100644 --- a/crates/types/src/ir/schema/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -12,14 +12,18 @@ use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; /// Where one output column of an aggregate comes from: an input column it -/// keeps, a measure's value, or a label column the measure adds -/// (`count_values`' and `absent`'s). `sum by (job) (x)` → -/// `[InputColumn(job), Measure(0)]`. The one layout the output schema +/// keeps, a measure's value (with the input column the schema types it by, +/// the measure's own or the probe value column), or a label column the +/// measure adds (`count_values`' and `absent`'s). `sum by (job) (x)` → +/// `[InputColumn(job), Measure { index: 0, input_column: Some(value) }]`. The one layout the output schema /// ([`aggregate_output_schema`]) and the group properties read. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OutputColumnSource { InputColumn(ColumnId), - Measure(usize), + Measure { + index: usize, + input_column: Option, + }, AddedLabel, } /// Output schema of a *per-series* window/range reduction (`rate`/`increase`, @@ -49,7 +53,10 @@ fn per_series_reduction_schema( let mut sources: Vec = (0..columns.len()) .map(OutputColumnSource::InputColumn) .collect(); - sources[vi] = OutputColumnSource::Measure(0); + sources[vi] = OutputColumnSource::Measure { + index: 0, + input_column: Some(vi), + }; { let mut out = agg.output_column(&columns[vi]); out.name = "value".into(); @@ -104,7 +111,7 @@ pub fn aggregate_output_columns( } /// [`aggregate_output_schema`] and [`aggregate_output_columns`] in one pass. -fn aggregate_output( +pub(crate) fn aggregate_output( in_schema: &Schema, reduction: &Reduction, measures: &[AggIntent], @@ -174,7 +181,10 @@ fn aggregate_output( cnt.name = name.clone(); } out_cols.push(cnt); - sources.push(OutputColumnSource::Measure(i)); + sources.push(OutputColumnSource::Measure { + index: i, + input_column: value_col_idx, + }); continue; } // `absent(v)` emits its synthesized labels as `Utf8` columns before @@ -190,11 +200,8 @@ fn aggregate_output( // Only the output *type* is read from here, so the leading column is // enough for the multi-column intents: `Cardinality` and `PearsonCorr` // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); + let in_col = + measure_input_column(in_schema, intent).map_or(&probe, |id| &in_schema.fields[id]); let mut out = intent.output_column(in_col); // A global extremum emits NULL for an empty input, even if its input // column is non-nullable. Grouped extrema only emit existing groups. @@ -212,7 +219,10 @@ fn aggregate_output( out.name = name.clone(); } out_cols.push(out); - sources.push(OutputColumnSource::Measure(i)); + sources.push(OutputColumnSource::Measure { + index: i, + input_column: measure_input_column(in_schema, intent).or(value_col_idx), + }); } // `count_values` groups by (by-keys ∪ the synthesized value label), so the // by-keys alone are not a unique key — be conservative and claim none. @@ -238,6 +248,17 @@ fn aggregate_output( )) } +/// The input column a measure's output is typed by when it names one: its +/// leading input column (the multi-column intents' output type ignores the +/// rest). +fn measure_input_column(in_schema: &Schema, intent: &AggIntent) -> Option { + intent + .input_cols() + .first() + .copied() + .filter(|&id| id < in_schema.fields.len()) +} + /// Output schema of a `without(excluded)` aggregate: the kept labels (every /// input label column except the `excluded` positions, the time axis, and the /// sample-value column) followed by the aggregate output column(s). Unlike the @@ -282,11 +303,8 @@ fn without_output_schema( // Only the output *type* is read from here, so the leading column is // enough for the multi-column intents: `Cardinality` and `PearsonCorr` // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); + let in_col = + measure_input_column(in_schema, intent).map_or(&probe, |id| &in_schema.fields[id]); let mut out = intent.output_column(in_col); if let Some((arg, _)) = intent .arg_selector_columns(in_schema) @@ -299,7 +317,11 @@ fn without_output_schema( out.name = name.clone(); } out_cols.push(out); - sources.push(OutputColumnSource::Measure(i)); + sources.push(OutputColumnSource::Measure { + index: i, + input_column: measure_input_column(in_schema, intent) + .or_else(|| in_schema.column_id("value")), + }); } Ok(( Schema { @@ -347,7 +369,13 @@ mod tests { use OutputColumnSource::*; assert_eq!( columns(Reduction::by(vec![2]), &[AggIntent::Sum { col: None }]), - vec![InputColumn(2), Measure(0)] + vec![ + InputColumn(2), + Measure { + index: 0, + input_column: Some(1) + } + ] ); } @@ -360,7 +388,13 @@ mod tests { Reduction::Reduce(GroupKeys::without(vec![3])), &[AggIntent::Sum { col: None }] ), - vec![InputColumn(2), Measure(0)] + vec![ + InputColumn(2), + Measure { + index: 0, + input_column: Some(1) + } + ] ); } @@ -374,14 +408,27 @@ mod tests { }; assert_eq!( columns(Reduction::by(vec![2]), std::slice::from_ref(&count_values)), - vec![InputColumn(2), AddedLabel, Measure(0)] + vec![ + InputColumn(2), + AddedLabel, + Measure { + index: 0, + input_column: Some(1) + } + ] ); assert_eq!( columns( Reduction::Reduce(GroupKeys::without(vec![3])), &[count_values] ), - vec![InputColumn(2), Measure(0)] + vec![ + InputColumn(2), + Measure { + index: 0, + input_column: Some(1) + } + ] ); } @@ -398,7 +445,14 @@ mod tests { }; assert_eq!( columns(Reduction::by(vec![]), &[absent]), - vec![AddedLabel, AddedLabel, Measure(0)] + vec![ + AddedLabel, + AddedLabel, + Measure { + index: 0, + input_column: Some(1) + } + ] ); } @@ -409,7 +463,15 @@ mod tests { use OutputColumnSource::*; assert_eq!( columns(Reduction::PerEntity, &[AggIntent::Rate]), - vec![InputColumn(0), Measure(0), InputColumn(2), InputColumn(3)] + vec![ + InputColumn(0), + Measure { + index: 0, + input_column: Some(1) + }, + InputColumn(2), + InputColumn(3) + ] ); } diff --git a/crates/types/src/ir/schema/error.rs b/crates/types/src/ir/schema/error.rs index d4f3b05b5..4aa37e4ce 100644 --- a/crates/types/src/ir/schema/error.rs +++ b/crates/types/src/ir/schema/error.rs @@ -17,5 +17,5 @@ pub enum SchemaDerivationError { #[error("invalid per-series sample column: {0}")] InvalidSampleColumn(String), #[error("invalid summary coverage: {0}")] - Coverage(#[from] crate::ir::properties::summary_coverage::CoverageError), + Coverage(#[from] crate::ir::properties::aggregate_coverage::CoverageError), } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 5a29d61e6..9436c94bf 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,6 +1,5 @@ //! Ranked-row output contract shared by logical TopK and its implementations. -use crate::ir::operator::arguments::Source; -use crate::ir::operator::input::{scanned_sources, OperatorInput}; +use crate::ir::operator::input::{scans_only_time_series, OperatorInput}; use crate::ir::operator::Reduction; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; @@ -56,20 +55,6 @@ pub fn ranked_column(child: &I) -> Result(input: &I) -> bool { - fn sources_below(input: &I) -> std::collections::BTreeSet { - scanned_sources(&input.operator(), sources_below) - } - sources_below(input) - .iter() - .all(|source| matches!(source, Source::TimeSeries { .. })) -} - -/// Ordered projection of ranked rows: partition keys, ranked value, then -/// item identity. TopK retains rows; partition keys alone are not unique. pub(crate) fn ranked_row_projection( child: &I, reduction: &Reduction, @@ -98,7 +83,7 @@ pub(crate) fn ranked_row_projection( alias: Some(output_name.into()), expr: ScalarExpr::Column(value), }); - let promql = scans_only_time_series(child); + let promql = scans_only_time_series(&child.scanned_sources()); for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item diff --git a/crates/types/src/plan/materialized_view.rs b/crates/types/src/plan/materialized_view.rs index dd8d51737..3c8f49484 100644 --- a/crates/types/src/plan/materialized_view.rs +++ b/crates/types/src/plan/materialized_view.rs @@ -76,8 +76,8 @@ use crate::ir::operator::arguments::Source; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::{Operator, OperatorNode}; use crate::ir::operator::non_asap::NonASAPOp; +use crate::ir::properties::aggregate_coverage::{AggregateCoverage, CoverageError, SelectionBox}; use crate::ir::properties::execution_data_state::ExecutionTiming; -use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, SummaryCoverage}; use crate::stages::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is @@ -120,12 +120,12 @@ impl MaterializedView { } /// A view's summary coverage -/// ([`SummaryCoverage`], `docs/design_docs/proposals/asap-primitive-schema.md` +/// ([`AggregateCoverage`], `docs/design_docs/proposals/asap-primitive-schema.md` /// §4.2): what it computes and which rows went in. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ViewCoverage { - /// `SummaryCoverage::definition_id`: equal on every node, plan and + /// `AggregateCoverage::definition_id`: equal on every node, plan and /// process that computes the same thing, whatever its selection; a /// rollup to a coarser grouping computes something else and has its /// own. @@ -138,7 +138,7 @@ pub struct ViewCoverage { #[cfg(any(test, feature = "test-support"))] impl ViewCoverage { - /// `SummaryCoverage::definition_id`. + /// `AggregateCoverage::definition_id`. pub fn definition_id(&self) -> &str { &self.definition_id } @@ -600,7 +600,7 @@ impl PhysicalASAPDAG { self.operator_nodes()? }; let mut derived = Vec::new(); - let mut coverage_by_node: HashMap = HashMap::new(); + let mut coverage_by_node: HashMap = HashMap::new(); for &id in index.children_first.iter().filter(|id| views.contains(id)) { let node = index.nodes[&id]; let (inputs, scans) = index.inputs_of_view(id, &views)?; @@ -931,8 +931,8 @@ fn coverage_of( id: PhysicalASAPNodeId, inputs: &MaterializedViewInputs, ir: &HashMap>, - coverage_by_node: &HashMap, -) -> Result { + coverage_by_node: &HashMap, +) -> Result { let _ = (id, inputs, ir, coverage_by_node); todo!( "#1207 PR 6: the view's coverage from its memo group's GroupProperties::aggregate_coverage" diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index ebb2d33ed..b6b126da7 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -258,7 +258,7 @@ impl PhysicalASAPDAG { /// Every node of this DAG as IR, by node id, with its exported /// guarantee (an operator node has no physical properties): what a caller derives /// IR properties of a plan file's node from, such as its - /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). + /// [`AggregateCoverage`](crate::ir::properties::aggregate_coverage::AggregateCoverage). /// Shared inputs stay shared. Validates the DAG first. pub(crate) fn operator_nodes( &self, diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index 18accf1e5..57792c18a 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -1,4 +1,4 @@ -//! Aggregate coverage, a group property: [`SummaryCoverage`] derived once +//! Aggregate coverage, a group property: [`AggregateCoverage`] derived once //! for each aggregate memo group from its input groups. //! //! It is derived for a summary state or an exact `Aggregate`, never @@ -18,22 +18,22 @@ use std::ops::Bound; use super::MemoGroupAsInput; use crate::ir::operator::arguments::{AtModifier, Reduction}; -use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; +use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::properties::summary_coverage::{ - dedup, interval, union, ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, - SelectionBox, SummaryCoverage, SummaryStateDefinition, +use crate::ir::properties::aggregate_coverage::{ + constraint_of, union, AggregateCoverage, AggregateDefinition, ColumnIdentity, Constraint, + CoverageError, IntervalConstraintLimit, SelectionBox, }; -use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::expr_ir::ScalarValue; use crate::ir::scalar::{Predicate, ScalarExpr}; use crate::ir::schema::{ColumnId, Schema}; use crate::ir::SchemaDerivationError; use crate::stages::MemoGroupId; use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; -impl SummaryCoverage { +impl AggregateCoverage { /// Coverage of what a memo group computes with `operator` over its /// input groups: a `SummaryAgg`, an exact `Aggregate`, a /// `SummaryTumblingWindowChain` (a build over its input, the timestamp @@ -48,7 +48,7 @@ impl SummaryCoverage { /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may /// also merge overlapping selections, and a counter family /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. - pub(crate) fn derive_from_inputs<'m>( + pub(super) fn derive_from_inputs<'m>( operator: &Operator>, ) -> Result { let as_built = |op: &ASAPOp>| -> ASAPOp> { @@ -67,7 +67,7 @@ impl SummaryCoverage { } Operator::NonASAP(_) => match selected_build(operator)? { Some(build) => of_summary_agg(&build), - None => Err(CoverageError::NotSummary), + None => Err(CoverageError::NoCoverage), }, Operator::ASAP( op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), @@ -81,7 +81,7 @@ impl SummaryCoverage { child .aggregate_coverage() .cloned() - .ok_or(CoverageError::NotSummary) + .ok_or(CoverageError::NoCoverage) }) .collect::, _>>()?; let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; @@ -90,7 +90,7 @@ impl SummaryCoverage { Some(keys) => regrouped(merged, &children[0], keys), } } - Operator::ASAP(_) => Err(CoverageError::NotSummary), + Operator::ASAP(_) => Err(CoverageError::NoCoverage), } } } @@ -124,10 +124,10 @@ fn selected_build<'m>( filter, }) = states.operator() else { - return Err(CoverageError::NotSummary); + return Err(CoverageError::NoCoverage); }; if by.is_without() { - return Err(CoverageError::NotSummary); + return Err(CoverageError::NoCoverage); } // Each state key column is the input column of the same name it groups by. let mut unmapped = None; @@ -154,7 +154,7 @@ fn selected_build<'m>( }) .map_err(regrouped_build)?; let reduction = Reduction::Reduce(by); - summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; + summary_agg_layout(rows.schema(), &family, &reduction).map_err(regrouped_build)?; Ok(Some(ASAPOp::SummaryAgg { child: rows, summary_type: family, @@ -167,9 +167,9 @@ fn selected_build<'m>( /// The coverage of a merge of states covering `inputs`, whose builds read /// rows with timestamp column `time_column`. fn of_merge( - inputs: &[SummaryCoverage], + inputs: &[AggregateCoverage], time_column: Option<&ColumnIdentity>, -) -> Result { +) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; if inputs .iter() @@ -211,7 +211,7 @@ fn of_merge( { return Err(CoverageError::InputsOverlap { required }); } - Ok(SummaryCoverage { + Ok(AggregateCoverage { definition: first.definition.clone(), selection: union(inputs.iter().flat_map(|c| c.selection.clone()).collect()), }) @@ -224,10 +224,10 @@ fn of_merge( /// of one build, so their rows are disjoint. A counter family merges only /// time-disjoint windows, never different groups at one time. fn regrouped( - merged: SummaryCoverage, + merged: AggregateCoverage, first: &MemoGroupAsInput<'_>, keys: &[ColumnId], -) -> Result { +) -> Result { let Some(Operator::ASAP(ASAPOp::SummaryAgg { summary_type: family, input, @@ -236,13 +236,13 @@ fn regrouped( .. })) = merged.definition.unlifted_operators.first() else { - return Err(CoverageError::NotSummary); + return Err(CoverageError::NoCoverage); }; if !SummaryTypeCapability::merges_groups(family) { return Err(CoverageError::SummaryTypeCannotRegroup); } let state = first.schema(); - let rows = build_input_schema(first).ok_or(CoverageError::NotSummary)?; + let rows = build_input_schema(first).ok_or(CoverageError::NoCoverage)?; // The build's input column each key of the state names. let mapped = keys .iter() @@ -268,7 +268,7 @@ fn regrouped( }) .collect::, _>>()?; let reduction = Reduction::by(mapped); - summary_agg_schema(&rows, family, &reduction).map_err(|error| { + summary_agg_layout(&rows, family, &reduction).map_err(|error| { CoverageError::RegroupedBuild { reason: error.to_string(), } @@ -282,7 +282,7 @@ fn regrouped( reduction, filter: filter.clone(), }); - Ok(SummaryCoverage { + Ok(AggregateCoverage { definition, selection: merged.selection, }) @@ -317,7 +317,7 @@ fn time_column_of_build_input(state: &I) -> Option>) -> Result { +fn of_summary_agg(build: &ASAPOp>) -> Result { let (ASAPOp::SummaryAgg { child, summary_type: family, @@ -355,7 +355,7 @@ fn of_aggregate( child: &GroupInput<'_>, filter: Option<&Predicate>>, unlifted_top: impl FnOnce(Option>>) -> Operator>, -) -> Result { +) -> Result { // The operators a predicate can be lifted through, top to bottom, and // the first input below them. A subquery's group stays named in the // residual of the operator that reads it. @@ -466,8 +466,8 @@ fn of_aggregate( .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) .collect(), }; - Ok(SummaryCoverage { - definition: SummaryStateDefinition { + Ok(AggregateCoverage { + definition: AggregateDefinition { input_group, unlifted_operators, }, @@ -624,7 +624,7 @@ fn time_constraint(links: &[Operator]) -> Option { /// select on the timestamp column of the chain's input, in every box. fn of_window_selection( selection: &Operator>, -) -> Result { +) -> Result { let mut links = vec![selection.clone()]; let mut below = *selection.children()[0]; while let Some(child) = selected_from(&below) { @@ -643,132 +643,26 @@ fn of_window_selection( Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ) { - return Err(CoverageError::NotSummary); + return Err(CoverageError::NoCoverage); } let chain = below .aggregate_coverage() - .ok_or(CoverageError::NotSummary)?; - let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NotSummary)?; - let constraint = time_constraint(&links).ok_or(CoverageError::NotSummary)?; + .ok_or(CoverageError::NoCoverage)?; + let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NoCoverage)?; + let constraint = time_constraint(&links).ok_or(CoverageError::NoCoverage)?; let selection = chain .selection .iter() .map(|selection_box| { let mut columns = selection_box.columns.clone(); let combined = combined_with(columns.get(&time_column), constraint.clone()) - .ok_or(CoverageError::NotSummary)?; + .ok_or(CoverageError::NoCoverage)?; columns.insert(time_column.clone(), combined); Ok(SelectionBox { columns }) }) .collect::>()?; - Ok(SummaryCoverage { + Ok(AggregateCoverage { definition: chain.definition.clone(), selection, }) } - -/// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of -/// equalities on one column, with non-null literals. -fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { - let literal = |e: &ScalarExpr| match e { - ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), - _ => None, - }; - match expr { - ScalarExpr::Compare { - left, op, right, .. - } => { - let (column, value, op) = match (left.as_ref(), right.as_ref()) { - (ScalarExpr::Column(c), r) => (*c, literal(r)?, op.clone()), - (l, ScalarExpr::Column(c)) => (*c, literal(l)?, flip(op)?), - _ => return None, - }; - let constraint = match op { - CompareOpKind::Eq => Constraint::In(vec![value]), - CompareOpKind::Ne => Constraint::NotIn(vec![value]), - CompareOpKind::Lt => interval(Bound::Unbounded, Bound::Excluded(value)), - CompareOpKind::Le => interval(Bound::Unbounded, Bound::Included(value)), - CompareOpKind::Gt => interval(Bound::Excluded(value), Bound::Unbounded), - CompareOpKind::Ge => interval(Bound::Included(value), Bound::Unbounded), - _ => return None, - }; - Some((column, constraint)) - } - ScalarExpr::InList { - expr, - list, - negated, - } => { - let ScalarExpr::Column(column) = expr.as_ref() else { - return None; - }; - let values = list.iter().map(literal).collect::>>()?; - let values = dedup(values); - Some(( - *column, - if *negated { - Constraint::NotIn(values) - } else { - Constraint::In(values) - }, - )) - } - ScalarExpr::BoolOr(disjuncts) => { - let mut column = None; - let mut values = Vec::new(); - for disjunct in disjuncts { - let (c, Constraint::In(v)) = constraint_of(disjunct)? else { - return None; - }; - if column.replace(c).is_some_and(|prev| prev != c) { - return None; - } - values.extend(v); - } - Some((column?, Constraint::In(dedup(values)))) - } - _ => None, - } -} - -fn flip(op: &CompareOpKind) -> Option { - Some(match op { - CompareOpKind::Eq => CompareOpKind::Eq, - CompareOpKind::Ne => CompareOpKind::Ne, - CompareOpKind::Lt => CompareOpKind::Gt, - CompareOpKind::Le => CompareOpKind::Ge, - CompareOpKind::Gt => CompareOpKind::Lt, - CompareOpKind::Ge => CompareOpKind::Le, - _ => return None, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - use crate::ir::scalar::ExprSemantics; - - use IntervalConstraintLimit::FixedValue; - - /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. - #[test] - fn a_value_comparison_is_an_interval_between_fixed_values() { - let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(0)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), - semantics: ExprSemantics::Sql, - }; - assert_eq!( - constraint_of(&price_gt_10), - Some(( - 0, - Constraint::Interval { - lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), - upper: Bound::Unbounded, - } - )) - ); - } -} diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs index 06919793b..6d62d4784 100644 --- a/crates/types/src/stages/logical/candidate.rs +++ b/crates/types/src/stages/logical/candidate.rs @@ -69,7 +69,7 @@ pub enum CandidateError { /// output schema does not derive, an input fails validation, or a merge /// reads states it may not merge): a bug in the rule that built it. #[error("the candidate's expression is not valid over its input groups: {0}")] - Schema(SchemaDerivationError), + NotValidOverInputGroups(SchemaDerivationError), /// Its output schema or output kind differs from the group's: it does /// not compute the group's result (a bug in the rule that built it). #[error("the candidate's output differs from group {group:?}'s")] diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index fa49b8759..0b2b6d1a7 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -7,7 +7,7 @@ use std::collections::BTreeMap; use super::{MemoGroupAsInput, ValueSign}; use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction}; -use crate::ir::operator::asap::{summary_agg_columns, summary_agg_schema, summary_build}; +use crate::ir::operator::asap::{summary_agg_layout, summary_build}; use crate::ir::operator::input::OperatorInput; use crate::ir::operator::AggIntent; use crate::ir::scalar::expr_ir::ScalarValue; @@ -23,8 +23,8 @@ use ValueSign::{NonNegative, NonPositive, Unknown}; type Input<'m> = MemoGroupAsInput<'m>; /// The sign of each column of `output`, the output schema of `operator`, -/// from its input groups' `column_sign` and, for a `Scan`, `source_facts`. -pub(crate) fn output_column_sign( +/// from its input groups' `column_signs` and, for a `Scan`, `source_facts`. +pub(super) fn output_column_sign( operator: &Operator>, output: &Schema, source_facts: &BTreeMap, @@ -43,7 +43,7 @@ pub(crate) fn output_column_sign( } fn signs_of(input: &Input<'_>) -> Vec { - input.group_properties().column_sign.clone() + input.group_properties().column_signs.clone() } fn non_asap_signs( @@ -158,10 +158,12 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { match op { SummaryAgg { child, + summary_type, input, reduction, .. - } => state_signs(child, input, reduction, output), + } => state_signs(child, summary_type, input, reduction) + .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs), // The build's layout without its own time column, after the // computed `window_end` (column 0). SummaryTumblingWindowChain { @@ -170,16 +172,15 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { input, reduction, .. - } => match summary_agg_schema(child.schema(), summary_type, reduction) { - Ok(build) => { - let mut signs = state_signs(child, input, reduction, &build); + } => match state_signs(child, summary_type, input, reduction) { + Some((build, mut signs)) => { if let Some(time) = build.time_index { signs.remove(time); } signs.insert(0, Unknown); signs } - Err(_) => vec![Unknown; output.fields.len()], + None => vec![Unknown; output.fields.len()], }, SummaryMerge { children, group_by } => { // The windows a `TimeRange` over a chain selects merge into the @@ -187,12 +188,14 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { if let (None, [child]) = (group_by, children.as_slice()) { if let Some(SummaryTumblingWindowChain { child, + summary_type, input, reduction, .. }) = summary_build(child) { - return state_signs(&child, &input, &reduction, output); + return state_signs(&child, &summary_type, &input, &reduction) + .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs); } } let merged = agreed(&children.iter().map(signs_of).collect::>()); @@ -275,9 +278,8 @@ fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { /// An aggregate's, by where each output column comes from /// ([`aggregate_output_columns`]): a kept input column keeps its sign, a -/// measure's value has [`measure_sign`] of its input value's (its first -/// input column, else the column `aggregate_output_schema` types it by), and -/// a label a measure adds is `Unknown`. +/// measure's value has [`measure_sign`] of its input column's, and a label +/// a measure adds is `Unknown`. fn aggregate_signs( child: &Input<'_>, reduction: &Reduction, @@ -287,48 +289,33 @@ fn aggregate_signs( let Ok(sources) = aggregate_output_columns(child.schema(), reduction, measures) else { return vec![Unknown; output.fields.len()]; }; - let input = signs_of(child); - signs_from_sources(child, &sources, |index, value| { - let measure = &measures[index]; - let value = measure.input_cols().first().copied().or(value); - measure_sign( - measure, - value - .and_then(|value| input.get(value).copied()) - .unwrap_or(Unknown), - ) + signs_from_sources(child, &sources, |index, input| { + measure_sign(&measures[index], input) }) } /// The sign of each output column from `sources`: an input column keeps -/// its sign, measure `index` has `measure(index, value)`, where `value` is -/// the column `aggregate_output_schema` types a measure without an input -/// column by (the sample value, else the first column not kept), and an -/// added label is `Unknown`. +/// its sign, measure `index` has `measure(index, the sign of its input +/// column)`, and an added label is `Unknown`. fn signs_from_sources( child: &Input<'_>, sources: &[OutputColumnSource], - measure: impl Fn(usize, Option) -> ValueSign, + measure: impl Fn(usize, ValueSign) -> ValueSign, ) -> Vec { - let input = signs_of(child); - let schema = child.schema(); - let kept: Vec = sources - .iter() - .filter_map(|source| match source { - OutputColumnSource::InputColumn(column) => Some(*column), - _ => None, - }) - .collect(); - let value = resolve_column_ref(&ColumnRef::SampleValue, schema) - .ok() - .or_else(|| (0..schema.fields.len()).find(|column| !kept.contains(column))); + let input = &child.group_properties().column_signs; + let sign_of = |column: Option| { + column + .and_then(|column| input.get(column).copied()) + .unwrap_or(Unknown) + }; sources .iter() .map(|source| match source { - OutputColumnSource::InputColumn(column) => { - input.get(*column).copied().unwrap_or(Unknown) - } - OutputColumnSource::Measure(index) => measure(*index, value), + OutputColumnSource::InputColumn(column) => sign_of(Some(*column)), + OutputColumnSource::Measure { + index, + input_column, + } => measure(*index, sign_of(*input_column)), OutputColumnSource::AddedLabel => Unknown, }) .collect() @@ -366,18 +353,17 @@ fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { } } -/// A summary state's, in its layout ([`summary_agg_columns`]): its key -/// columns keep their input's sign and its state column records the sign of -/// the values it is updated with. +/// A summary state's schema and signs, in its layout +/// ([`summary_agg_layout`]): its key columns keep their input's sign and its +/// state column records the sign of the values it is updated with; `None` +/// when the layout does not derive. fn state_signs( child: &Input<'_>, + summary_type: &FieldDataType, update: &SummaryUpdate, reduction: &Reduction, - state_schema: &Schema, -) -> Vec { - let Ok(sources) = summary_agg_columns(child.schema(), reduction) else { - return vec![Unknown; state_schema.fields.len()]; - }; +) -> Option<(Schema, Vec)> { + let (schema, sources) = summary_agg_layout(child.schema(), summary_type, reduction).ok()?; let updated_with = match &update.weight { SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) .ok() @@ -386,7 +372,8 @@ fn state_signs( SummaryInputExpr::Constant(value) => sign_of_number(*value), SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, }; - signs_from_sources(child, &sources, |_, _| updated_with) + let signs = signs_from_sources(child, &sources, |_, _| updated_with); + Some((schema, signs)) } /// A readout of summary state: the state's plain columns keep their sign diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 724089970..168f07040 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -5,8 +5,8 @@ use std::collections::BTreeSet; use super::column_properties::output_column_sign; use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; -use crate::ir::operator::input::{scanned_sources, OperatorInput}; -use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::operator::input::{sources_scanned_by, OperatorInput}; +use crate::ir::properties::aggregate_coverage::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; @@ -64,7 +64,7 @@ pub struct GroupProperties { pub result_kind: OperatorResultKind, /// Per output column, in schema order: the sign of its values, from the /// input groups' and, for a `Scan`, the declared ones. - pub column_sign: Vec, + pub column_signs: Vec, /// Whether the result changes with the evaluation time: true for a /// group under a `TimeRange` relative to the evaluation time /// (`sum_over_time(m[5m])`) or reading it (`time()`, `now()`), false @@ -75,7 +75,7 @@ pub struct GroupProperties { /// What an aggregate group covers: a summary state, a chain, a /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact /// `Aggregate` result; `None` for any other group. - pub aggregate_coverage: Option, + pub aggregate_coverage: Option, /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). pub shareable_when_equal: bool, @@ -84,16 +84,6 @@ pub struct GroupProperties { pub scanned_sources: BTreeSet, } -impl GroupProperties { - /// Whether every scanned source is a time series (PromQL rows): a - /// PromQL top-k ranks one row per series. - pub fn scans_only_time_series(&self) -> bool { - self.scanned_sources - .iter() - .all(|source| matches!(source, Source::TimeSeries { .. })) - } -} - /// The sign of one column's values, a group property derived from the /// input groups like lineage (concepts doc, section 3, "Value sign"); a /// `Scan`'s are declared per column in the data workload's source facts. @@ -109,21 +99,17 @@ pub enum ValueSign { /// Whether `operator` is a valid expression over its input groups /// ([`MemoGroupAsInput`]), never a node tree: its output schema derives, its /// inputs pass validation, and a merge reads only states it may merge. -/// Returns the output schema and the aggregate coverage. +/// Returns the output schema. pub(super) fn validate_expression_over_input_groups( operator: &Operator>, -) -> Result<(Schema, Option), SchemaDerivationError> { +) -> Result { let schema = operator.output_schema()?; operator.validate_inputs()?; - let aggregate_coverage = match SummaryCoverage::derive_from_inputs(operator) { - Ok(coverage) => Some(coverage), - // A merge is valid only over states of one definition it may merge. - Err(error) if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) => { - return Err(error.into()) - } - Err(_) => None, - }; - Ok((schema, aggregate_coverage)) + // A merge is valid only over states of one definition it may merge. + if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) { + AggregateCoverage::derive_from_inputs(operator)?; + } + Ok(schema) } /// The properties of a group computing `expression` over its input @@ -135,17 +121,16 @@ pub(super) fn derive_group_properties( expression: &Operator, ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); - let (schema, aggregate_coverage) = validate_expression_over_input_groups(&operator)?; + let schema = validate_expression_over_input_groups(&operator)?; + let aggregate_coverage = AggregateCoverage::derive_from_inputs(&operator).ok(); Ok(GroupProperties { - column_sign: output_column_sign(&operator, &schema, &memo.source_facts), + column_signs: output_column_sign(&operator, &schema, &memo.source_facts), depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), - scanned_sources: scanned_sources(&operator, |input| { - input.group_properties().scanned_sources.clone() - }), + scanned_sources: sources_scanned_by(&operator), }) } @@ -176,19 +161,19 @@ fn depends_on_evaluation_time(operator: &Operator>) -> bool ) => filter.iter().any(|pred| reads_evaluation_time(&pred.0)), Operator::ASAP(_) => false, }; - reads_it || operator.children().into_iter().any(depends_on_it) -} - -fn depends_on_it(input: &MemoGroupAsInput<'_>) -> bool { - input.group_properties().depends_on_evaluation_time + reads_it + || operator + .children() + .into_iter() + .any(|input| input.group_properties().depends_on_evaluation_time) } /// A memo group read as an operator's input: its `group_properties` and /// its `expression`, whose inputs are read the same way. #[derive(Clone, Copy)] -pub(crate) struct MemoGroupAsInput<'m> { - pub(crate) memo: &'m Memo, - pub(crate) group_id: MemoGroupId, +pub(super) struct MemoGroupAsInput<'m> { + pub(super) memo: &'m Memo, + pub(super) group_id: MemoGroupId, } impl<'m> MemoGroupAsInput<'m> { @@ -196,7 +181,7 @@ impl<'m> MemoGroupAsInput<'m> { &self.memo.groups()[self.group_id.0 as usize] } - pub(crate) fn group_properties(&self) -> &'m GroupProperties { + pub(super) fn group_properties(&self) -> &'m GroupProperties { &self.group().group_properties } } @@ -217,7 +202,11 @@ impl OperatorInput for MemoGroupAsInput<'_> { }) } - fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { self.group().group_properties.aggregate_coverage.as_ref() } + + fn scanned_sources(&self) -> BTreeSet { + self.group().group_properties.scanned_sources.clone() + } } diff --git a/crates/types/src/stages/logical/memo/hash_consing.rs b/crates/types/src/stages/logical/memo/hash_consing.rs index a7c4f26ef..78c68be6f 100644 --- a/crates/types/src/stages/logical/memo/hash_consing.rs +++ b/crates/types/src/stages/logical/memo/hash_consing.rs @@ -1,5 +1,5 @@ //! The one equality that makes two expressions one memo group, and two -//! candidates of a group one candidate: [`same_expression`] (operator, +//! candidates of a group one candidate: [`same_serialized_expression`] (operator, //! arguments and input groups). [`Memo::find_or_add_group`] looks a group up //! by it, Stage 0 included, and [`Memo::add_candidate`] finds a duplicate by //! it. Parameters θ are not an argument of any operator (they are a @@ -22,17 +22,6 @@ fn operator_and_arguments(expression: &Operator) -> Operator<()> { expression.map_children(|_| ()) } -/// Whether `left` and `right` are one expression: equal operators and -/// arguments ([`same_value`]) over the same input groups at the same -/// positions. -pub(super) fn same_expression(left: &Operator, right: &Operator) -> bool { - left.children() == right.children() - && same_value( - &operator_and_arguments(left), - &operator_and_arguments(right), - ) -} - /// The canonical JSON of `expression`'s operator and arguments, serialized /// once per expression: what [`expression_hash`] hashes and /// [`same_serialized_expression`] compares; `None` when it does not @@ -41,7 +30,7 @@ pub(super) fn serialized_operator(expression: &Operator) -> Option< serde_json::to_string(&operator_and_arguments(expression)).ok() } -/// A hash consistent with [`same_expression`], used only to narrow the +/// A hash consistent with [`same_serialized_expression`], used only to narrow the /// groups compared: SipHash over `serialized`, the expression's /// [`serialized_operator`], then the input groups. A non-finite `f64` /// serializes as `null`, which only widens one bucket; @@ -53,8 +42,10 @@ pub(super) fn expression_hash(expression: &Operator, serialized: Op hasher.finish() } -/// [`same_expression`] with each side's [`serialized_operator`] computed -/// already. +/// Whether `left` and `right` are one expression: equal operators and +/// arguments, by typed equality confirmed by equal serializations (each +/// side's [`serialized_operator`], computed once by the caller), over the +/// same input groups at the same positions. pub(super) fn same_serialized_expression( (left, left_serialized): (&Operator, Option<&str>), (right, right_serialized): (&Operator, Option<&str>), @@ -65,6 +56,14 @@ pub(super) fn same_serialized_expression( && left_serialized == right_serialized } +/// [`same_serialized_expression`], serializing both sides. +fn same_expression(left: &Operator, right: &Operator) -> bool { + same_serialized_expression( + (left, serialized_operator(left).as_deref()), + (right, serialized_operator(right).as_deref()), + ) +} + /// Typed equality, confirmed by equal serializations: numeric `PartialEq` /// alone conflates signed zeros, and `SketchParams`' equality omits the /// recorded error and failure probability. The serialized check is diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 8de0b4cae..101e89f3d 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -26,7 +26,7 @@ use hash_consing::{ /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with /// its candidates ([`MemoGroup::candidates`]: empty after Stage 0), and each /// query's root over them. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct Memo { /// The groups, by id ([`Self::groups`]); added only through /// [`Self::find_or_add_group`]. @@ -35,7 +35,7 @@ pub struct Memo { pub queries: Vec, /// The groups [`Self::find_or_add_group`] may return, by /// [`expression_hash`] of their expression: every group whose sharing - /// is legal ([`GroupProperties::shareable_when_equal`]). + /// is legal ([`GroupProperties::shareable_when_equal`](crate::stages::GroupProperties::shareable_when_equal)). groups_by_expression_hash: HashMap)>>, /// The data workload's facts per source (`DataWorkload::sources`), fixed /// at construction: a `Scan` group's column signs read them. @@ -59,11 +59,13 @@ pub struct MemoQuery { impl Memo { /// A memo without groups over the sources `data` describes: a `Scan` /// group's column signs are the ones `data` declares for its source - /// (`GroupProperties::column_sign`). + /// (`GroupProperties::column_signs`). pub fn new(data: &DataWorkload) -> Self { Self { + groups: Vec::new(), + queries: Vec::new(), + groups_by_expression_hash: HashMap::new(), source_facts: data.sources.clone(), - ..Self::default() } } @@ -85,10 +87,10 @@ impl Memo { /// Hash-consing: the group whose expression is `expression` /// (`same_expression`: operator, arguments and input groups), else a - /// new group for it, with the `GroupProperties` [`derive_group_properties`] + /// new group for it, with the `GroupProperties` `derive_group_properties` /// derives from its input groups, listed as a parent of each input /// group. A group whose copies may return different rows (an unordered - /// `LIMIT`, [`GroupProperties::shareable_when_equal`]) is never returned + /// `LIMIT`, [`GroupProperties::shareable_when_equal`](crate::stages::GroupProperties::shareable_when_equal)) is never returned /// for another expression, so such an expression always gets a new /// group. Fails, leaving the memo unchanged, when the expression is not /// valid over its input groups (its output schema does not derive, an @@ -148,7 +150,7 @@ impl Memo { /// /// Fails, leaving the memo unchanged, when the candidate's expression /// is not valid over its input groups - /// ([`validate_expression_over_input_groups`]), + /// (`validate_expression_over_input_groups`), /// or when its output schema or kind differs from the group's: then it /// does not compute the group's result. pub fn add_candidate( @@ -173,8 +175,8 @@ impl Memo { memo: self, group_id, }); - let (schema, _) = - validate_expression_over_input_groups(&operator).map_err(CandidateError::Schema)?; + let schema = validate_expression_over_input_groups(&operator) + .map_err(CandidateError::NotValidOverInputGroups)?; let result_kind = operator.output_kind(); let group_properties = &self.groups[group.0 as usize].group_properties; if schema != group_properties.schema || result_kind != group_properties.result_kind { diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 86f97e73d..8376f86d6 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -4,10 +4,13 @@ use super::*; use crate::ir::operator::arguments::{GroupKeys, Reduction, Source}; +use crate::ir::operator::AggIntent; use crate::ir::scalar::expr_ir::ColumnRef; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::{ProjectItem, SortKey}; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; use crate::ir::ASAPOp; -use crate::ir::NonASAPOp; +use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; use crate::stages::logical::MemoGroupAsInput; use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate}; @@ -106,7 +109,7 @@ fn kll_k(k: u32) -> Option { /// An equal expression over the same input groups is found, not added. #[test] fn find_or_add_group_returns_the_group_of_an_equal_expression() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); assert_eq!(memo.find_or_add_group(scan("latency")).unwrap(), g1); @@ -122,7 +125,7 @@ fn find_or_add_group_returns_the_group_of_an_equal_expression() { /// The same operator over different input groups is a different group. #[test] fn find_or_add_group_separates_equal_operators_over_different_inputs() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let latency = memo.find_or_add_group(scan("latency")).unwrap(); let errors = memo.find_or_add_group(scan("errors")).unwrap(); assert_ne!( @@ -135,7 +138,7 @@ fn find_or_add_group_separates_equal_operators_over_different_inputs() { /// `LIMIT`) gets a new group every time. #[test] fn find_or_add_group_never_returns_an_unshareable_group() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); assert_ne!( memo.find_or_add_group(unordered_limit(g1)).unwrap(), @@ -146,7 +149,7 @@ fn find_or_add_group_never_returns_an_unshareable_group() { /// A new group is a parent of each of its input groups, listed once. #[test] fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); memo.find_or_add_group(kll_state(g1)).unwrap(); @@ -158,7 +161,7 @@ fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { /// G2: θ is a logical property, not part of the expression. #[test] fn kll_builds_with_k_200_and_k_345_share_one_group() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let k200_group = memo.find_or_add_group(kll_state(g1)).unwrap(); let k200 = memo @@ -181,7 +184,7 @@ fn kll_builds_with_k_200_and_k_345_share_one_group() { /// in the group is not added again; its id is returned. #[test] fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); let first = memo @@ -208,7 +211,7 @@ fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { #[test] fn add_candidate_maintains_parents_and_queries() { // Query 0 is G3 = SummaryEstimate[q=0.95](G2) over G2 = KLL(G1). - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); @@ -243,7 +246,7 @@ fn add_candidate_maintains_parents_and_queries() { /// readout offered for a state group) is rejected and changes nothing. #[test] fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); let before = format!("{memo:?}"); @@ -262,7 +265,7 @@ fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { /// raw rows instead of summary state) is rejected and changes nothing. #[test] fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); @@ -277,7 +280,7 @@ fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { let before = format!("{memo:?}"); assert!(matches!( memo.add_candidate(g3, candidate(quantile_readout(g1, 0.95), None)), - Err(CandidateError::Schema(_)) + Err(CandidateError::NotValidOverInputGroups(_)) )); assert_eq!( format!("{memo:?}"), @@ -290,7 +293,7 @@ fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { /// is an error, and no group is added. #[test] fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let out_of_range = Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::by(vec![9]), @@ -304,3 +307,224 @@ fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { assert_eq!(memo.groups().len(), 1); assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); } + +/// `Scan m`: `[ts, service, value, latency]`, no unique key. +fn service_scan() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +/// Whether `a` and `b`, each built into one memo through +/// `find_or_add_group`, are one group. +fn one_group( + a: impl FnOnce(&mut Memo) -> MemoGroupId, + b: impl FnOnce(&mut Memo) -> MemoGroupId, +) -> bool { + let mut memo = Memo::new(&DataWorkload::default()); + let a = a(&mut memo); + let b = b(&mut memo); + a == b +} + +/// `operator` over the group of [`service_scan`]. +fn over_scan( + operator: impl FnOnce(MemoGroupId) -> Operator, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + |memo| { + let scan = memo.find_or_add_group(service_scan()).unwrap(); + memo.find_or_add_group(operator(scan)).unwrap() + } +} + +/// `quantile(q, col) by (by)` over [`service_scan`], with `filter`. +fn quantile_agg( + by: Vec, + col: Option, + q: f64, + filter: Option>, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(by), + measures: vec![AggIntent::Quantile { col, q }], + output_names: vec![], + filters: vec![filter], + having: None, + child, + }) + }) +} + +/// Aggregates that differ in a column, a quantile or a filter are +/// different groups. +#[test] +fn aggregates_that_differ_in_an_argument_are_different_groups() { + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(3), 0.5, None) + )); + assert!(!one_group( + quantile_agg(vec![], Some(2), 0.5, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let latency_above_1 = Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(3)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Sql, + }); + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(2), 0.5, Some(latency_above_1)) + )); +} + +/// Equal expressions without a unique key are one group when they return +/// the same rows each time: a keyless scan, a global aggregate, a +/// `without` aggregate. +#[test] +fn equal_keyless_expressions_that_return_the_same_rows_are_one_group() { + let table_scan = |memo: &mut Memo| { + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "metrics".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, true), + ], + 0, + vec![], + ), + })) + .unwrap() + }; + assert!(one_group(table_scan, table_scan)); + assert!(one_group( + quantile_agg(vec![], Some(2), 0.9, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let without = || { + over_scan(|child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::without(vec![0])), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }) + }; + assert!(one_group(without(), without())); +} + +/// `DISTINCT` on every column returns the same rows each time and is one +/// group; a `Dedup` on some columns keeps an unspecified row per key. +#[test] +fn dedup_is_one_group_only_on_every_column() { + let dedup = |cols: Vec| { + over_scan(move |child| Operator::NonASAP(NonASAPOp::Dedup { cols, child })) + }; + assert!(one_group(dedup(vec![]), dedup(vec![]))); + assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); +} + +/// Two unordered `LIMIT 10`s may return different rows: two groups, and +/// so are equal parents over them. +#[test] +fn unordered_limits_and_their_parents_are_different_groups() { + assert!(!one_group( + over_scan(unordered_limit), + over_scan(unordered_limit) + )); + let distinct_over_limit = |memo: &mut Memo| { + let limit = over_scan(unordered_limit)(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Dedup { + cols: vec![], + child: limit, + })) + .unwrap() + }; + assert!(!one_group(distinct_over_limit, distinct_over_limit)); +} + +/// A `LIMIT` over a `Sort` with the same partitions keeps the first rows +/// of a fixed order (one group); over a `Sort` partitioned differently, +/// which rows come first depends on input order (two groups). +#[test] +fn limits_over_a_sort_with_their_partitions_are_one_group() { + let limit = |sort_partitions: GroupKeys| { + move |memo: &mut Memo| { + let sorted = over_scan(|child| { + Operator::NonASAP(NonASAPOp::Sort { + keys: vec![SortKey { + expr: ScalarExpr::Column(2), + ascending: false, + nulls_first: false, + }], + partition_by: sort_partitions, + child, + }) + })(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: sorted, + })) + .unwrap() + } + }; + assert!(one_group( + limit(GroupKeys::default()), + limit(GroupKeys::default()) + )); + assert!(!one_group( + limit(GroupKeys::by(vec![1])), + limit(GroupKeys::by(vec![1])) + )); +} + +/// A projection emitting the literal `value`. +fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Literal(ScalarValue::Float64(value)), + }], + qualifier: None, + child, + }) + }) +} + +/// Signed zeros and different non-finite literals are different groups; +/// JSON's `null` for non-finite floats never decides equality. +#[test] +fn signed_zero_and_non_finite_literals_are_different_groups() { + for (a, b) in [ + (0.0, -0.0), + (-0.0, 0.0), + (f64::INFINITY, f64::NEG_INFINITY), + (f64::NAN, f64::NAN), + ] { + assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); + } + assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); +} diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 280066481..9322812c3 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -15,7 +15,7 @@ pub use candidate::{ LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, }; pub use edge::LogicalEdge; -pub(crate) use group::MemoGroupAsInput; +use group::MemoGroupAsInput; pub use group::{GroupProperties, MemoGroup, MemoGroupId, ValueSign}; pub use memo::{queries_rooted_at, root_groups, Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 544114faf..502592071 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -110,6 +110,7 @@ mod tests { producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, Lifetime, PhysicalProperties, }; + use crate::workload::DataWorkload; use crate::plan::materialized_view::RefreshWindow; @@ -163,7 +164,7 @@ mod tests { /// 5 minutes, each with its own expression as its one candidate. Built /// through `find_or_add_group` and `add_candidate`. fn memo() -> Memo { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let scan = memo .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { @@ -284,7 +285,7 @@ mod tests { /// the one evaluation its result belongs to. Neither reads the memo. #[test] fn delivers_on_by_lifetime_without_windows() { - let candidates = candidates(Memo::default(), vec![]); + let candidates = candidates(Memo::new(&DataWorkload::default()), vec![]); let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index 9d28aa324..c389921c2 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -86,7 +86,7 @@ pub enum PruneReason { /// candidates, once each candidate's allowed consumers are applied. ReachesNoQueryOutput, /// Stage 2: a Count-Min build (`Cms`, `CmsWithHeap`, `HydraCms`) whose - /// weight column's `column_sign` is not `NonNegative`: its estimates are + /// weight column's `column_signs` is not `NonNegative`: its estimates are /// bounds only for non-negative weights. CountMinWeightsNotProvenNonNegative, /// Stage 3: the candidate it names, in the same memo group with the diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 3b3a314ae..cd646f4e8 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -6,7 +6,7 @@ use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; -use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::properties::aggregate_coverage::AggregateCoverage; use crate::plan::materialized_view::RawDataWindow; use crate::ir::schema::Schema; @@ -14,8 +14,8 @@ use crate::ir::{ASAPOp, NonASAPOp}; use crate::ir::{OperatorNode, SchemaDerivationError}; use std::rc::Rc; -/// `SummaryCoverage::definition_id`. -pub fn definition_id(coverage: &SummaryCoverage) -> String { +/// `AggregateCoverage::definition_id`. +pub fn definition_id(coverage: &AggregateCoverage) -> String { coverage.definition_id() } diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index aa9e55fc9..f7ea803bd 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -114,7 +114,7 @@ pub struct SourceFacts { pub metric_type: Option, /// The declared sign of each column's values, keyed by column name /// (default: none declared), e.g. `{"value": "non_negative"}` for a - /// latency gauge. A `Scan`'s `column_sign` reads it; an undeclared + /// latency gauge. A `Scan`'s `column_signs` reads it; an undeclared /// column is `Unknown`, except a PromQL counter's value, which is /// non-negative without a declaration. The deployment must ensure it. /// Omitted from JSON when empty. diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index 952ab82fd..7bce47c42 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,9 +2,9 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod memo_aggregate_coverage; mod memo_fixtures; mod memo_group_properties; -mod memo_summary_coverage; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; diff --git a/crates/types/tests/memo_summary_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs similarity index 96% rename from crates/types/tests/memo_summary_coverage.rs rename to crates/types/tests/memo_aggregate_coverage.rs index c2e61eeb0..830184c70 100644 --- a/crates/types/tests/memo_summary_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -14,8 +14,8 @@ use asap_types::ir::operator::arguments::{AtModifier, Reduction}; use asap_types::ir::operator::AggIntent; use crate::memo_fixtures::{filter, range, scan, shift, state}; -use asap_types::ir::properties::summary_coverage::{ - ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, SummaryCoverage, +use asap_types::ir::properties::aggregate_coverage::{ + AggregateCoverage, ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, }; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; @@ -29,6 +29,7 @@ use asap_types::stages::{ }; use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::definition_id; +use asap_types::workload::DataWorkload; const TS: usize = 0; const JOB: usize = 2; @@ -113,7 +114,7 @@ fn kll_of_last_minute_where( memo.find_or_add_group(kll(last_minute)).unwrap() } -fn coverage(memo: &Memo, group: MemoGroupId) -> &SummaryCoverage { +fn coverage(memo: &Memo, group: MemoGroupId) -> &AggregateCoverage { memo.groups()[group.0 as usize] .group_properties .aggregate_coverage @@ -164,7 +165,7 @@ fn time_of(memo: &Memo, group: MemoGroupId) -> Constraint { /// over G1. #[test] fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let coverage = coverage(&memo, g4); @@ -187,7 +188,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { /// `SummaryMerge(G4, G7)` is a valid group covering job ∈ {api, web}. #[test] fn states_with_equal_definitions_and_disjoint_selections_merge() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let g7 = kll_of_last_minute_where(&mut memo, g1, job_is("web")); @@ -215,7 +216,7 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { /// once per input. #[test] fn states_with_overlapping_selections_do_not_merge() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let groups = memo.groups().len(); @@ -234,7 +235,7 @@ fn states_with_overlapping_selections_do_not_merge() { /// definition; G10 does not merge with G4 although both select job = api. #[test] fn a_filter_that_is_no_selection_stays_unlifted() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let g10 = kll_of_last_minute_where( @@ -278,7 +279,7 @@ fn a_above(threshold: MemoGroupId) -> ScalarExpr { /// equal definition and disjoint selections merge. #[test] fn states_whose_definitions_read_the_same_subquery_merge() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let threshold = memo .find_or_add_group(Operator::NonASAP(NonASAPOp::Values { @@ -317,7 +318,7 @@ fn states_whose_definitions_read_the_same_subquery_merge() { /// without them, is the last unlifted operator. #[test] fn scan_predicates_lift_and_their_states_merge() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let state_of_job = |memo: &mut Memo, job: &str| { let series = memo.find_or_add_group(latency(vec![job_is(job)])).unwrap(); let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); @@ -356,7 +357,7 @@ fn scan_predicates_lift_and_their_states_merge() { /// ``` #[test] fn a_merge_over_a_chains_windows_covers_the_direct_build() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(shift(HOUR_MS, None, g1)).unwrap(); let g3 = memo.find_or_add_group(range(5 * MINUTE_MS, g2)).unwrap(); @@ -399,7 +400,7 @@ fn a_merge_over_a_chains_windows_covers_the_direct_build() { /// each under a `SummaryAgg[KLL]`. #[test] fn time_range_and_shifts_lift_into_one_interval_on_the_timestamp_column() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let plain = kll_over_last_5m(&mut memo, g1); let offset = kll_over_last_5m_of(&mut memo, shift(HOUR_MS, None, g1)); @@ -449,7 +450,7 @@ fn kll_over_last_5m_of(memo: &mut Memo, expression: Operator) -> Me /// ``` #[test] fn at_end_and_at_start_share_the_subtree_they_anchor_and_stay_unlifted() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); let g3 = memo.find_or_add_group(kll(g2)).unwrap(); @@ -480,7 +481,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { right: Box::new(ScalarExpr::Literal(value)), semantics: ExprSemantics::Sql, }; - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo .find_or_add_group(filter( @@ -528,7 +529,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { #[test] fn counter_states_merge_over_windows_disjoint_in_time() { let increase = || FieldDataType::exact(ExactKind::Increase); - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); let g3 = memo.find_or_add_group(state(increase(), g2)).unwrap(); @@ -569,7 +570,7 @@ fn an_exact_aggregate_has_coverage() { child, }) }; - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g2 = memo.find_or_add_group(filter(job_is("api"), g1)).unwrap(); let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); diff --git a/crates/types/tests/memo_fixtures.rs b/crates/types/tests/memo_fixtures.rs index 9ae9243d5..b64386ceb 100644 --- a/crates/types/tests/memo_fixtures.rs +++ b/crates/types/tests/memo_fixtures.rs @@ -1,4 +1,4 @@ -//! The memo expressions `memo_group_properties` and `memo_summary_coverage` +//! The memo expressions `memo_group_properties` and `memo_aggregate_coverage` //! build their examples from. use std::time::Duration; diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index a14bd6e14..fbec6f122 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -1,6 +1,7 @@ -//! `column_sign` and `depends_on_evaluation_time`, derived once per memo -//! group from its input groups (concepts doc, section 3, "Value sign" and -//! `GroupProperties`). Every example reads one of three sources: +//! `column_signs`, `depends_on_evaluation_time` and `scanned_sources`, +//! derived once per memo group from its input groups (concepts doc, section +//! 3, "Value sign" and `GroupProperties`; `aggregate_coverage` is in +//! `memo_aggregate_coverage.rs`). Every example reads one of three sources: //! //! ```text //! latency [ts, value, host] value declared non-negative @@ -10,6 +11,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::ir::operator::arguments::{AtModifier, GroupKeys, JoinKind, Reduction, Source}; +use asap_types::ir::operator::input::scans_only_time_series; use asap_types::ir::operator::AggIntent; use crate::memo_fixtures::{filter, range, scan, shift, state}; @@ -59,7 +61,9 @@ fn add(memo: &mut Memo, expression: Operator) -> MemoGroupId { } fn signs(memo: &Memo, group: MemoGroupId) -> &[ValueSign] { - &memo.groups()[group.0 as usize].group_properties.column_sign + &memo.groups()[group.0 as usize] + .group_properties + .column_signs } fn depends(memo: &Memo, group: MemoGroupId) -> bool { @@ -131,7 +135,7 @@ fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator /// Rows of the per-operator table that keep or set a column's sign: /// -/// | Group | Expression | `column_sign` | +/// | Group | Expression | `column_signs` | /// |---|---|---| /// | G1 | `Scan latency` | `[Unknown, NonNegative, Unknown]` (declared) | /// | G2 | `Scan gauge` | `[Unknown, Unknown, Unknown]` (nothing declared) | @@ -208,7 +212,7 @@ fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { /// Aggregates: keys keep their sign, sum / max keep the value's, counts are /// non-negative, and a rate is non-negative over a non-negative value only: /// -/// | Group | Expression | `column_sign` | +/// | Group | Expression | `column_signs` | /// |---|---|---| /// | G2 | `sum by (host)(G1)`, G1 = `Scan latency` | `[Unknown, NonNegative]` | /// | G3 | `max by (host)(G1)` | `[Unknown, NonNegative]` | @@ -262,7 +266,7 @@ fn aggregates_keep_their_keys_and_derive_their_values() { /// keeps the sign its inputs agree on, and a readout reads the state's, a /// count-like statistic being non-negative whatever the values: /// -/// | Group | Expression | `column_sign` | +/// | Group | Expression | `column_signs` | /// |---|---|---| /// | G3 | `SummaryAgg[KLL](G2)`, G2 = `latency[1m]` | state `NonNegative` | /// | G6 | `SummaryAgg[KLL](G5)`, G5 = `(latency offset 1m)[1m]` | state `NonNegative` | @@ -471,7 +475,7 @@ fn a_group_scans_the_sources_of_the_scans_below_it() { metric: "latency".into() }]) ); - let reads = |group: MemoGroupId| properties(group).scans_only_time_series(); + let reads = |group: MemoGroupId| scans_only_time_series(&properties(group).scanned_sources); assert!(reads(series)); assert!(reads(window)); assert!(!reads(requests)); @@ -497,8 +501,8 @@ fn count_values_without_has_one_sign_per_output_column() { ), ); let group = &memo.groups()[g2.0 as usize].group_properties; - assert_eq!(group.column_sign.len(), group.schema.fields.len()); - assert_eq!(group.column_sign.last(), Some(&NonNegative)); + assert_eq!(group.column_signs.len(), group.schema.fields.len()); + assert_eq!(group.column_signs.last(), Some(&NonNegative)); } /// A group's properties come from its operator and its input groups' @@ -506,7 +510,7 @@ fn count_values_without_has_one_sign_per_output_column() { /// readout's value column, and whether equal copies may be one group. #[test] fn group_properties_come_from_input_groups() { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let g1 = memo .find_or_add_group(scan( "latency", diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index b24531056..84cda9c5a 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -216,7 +216,7 @@ struct LabelSetDistinctValues { | `rows_per_second` | With `Ingesting`/`Mixed` | Rows arriving per second over all sources, used to price scans and continuous maintenance. It must be finite and non-negative. A source with no `series_count` + `sample_interval` of its own is priced at its series share of it when every source declares `series_count`, else at all of it. | | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | | `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | -| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan`'s `column_sign` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | +| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | | `distinct_values_per_label` | Optional; default empty; each count nonzero | Distinct values each label (PromQL) or column (SQL) takes, e.g. `{"host": 1000, "job": 10}`. Plan selection prices a `by (k1, k2, ...)` aggregate or summary, and a partitioned sort or limit, at the product of its keys' counts, at most the `series_count` (series, or groups of rows) of the sources it reads and the rows it reads per evaluation. Over several sources a key's counts add. No count is assumed for a key some source read does not declare: the grouping is priced at the smaller of that `series_count` and those rows. Each count must be an upper bound, counting a missing label (or a NULL) as one more value: when every key of a grouped result declares one, their product (at most the series read) also bounds `G`, which sizes per-group sketches for a whole-result target; otherwise `G` keeps its series bound, or none for SQL. | diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 156e0457f..d6d5479bb 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -117,7 +117,7 @@ A node in the physical data will represent the data or summary instance, so a no | How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | | How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `GroupProperties::aggregate_coverage`, `SummaryCoverage::derive_from_inputs` | §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, `AggregateCoverage::derive_from_inputs` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -237,11 +237,11 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), `SummaryCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::aggregate_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). +Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), `AggregateCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::aggregate_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). ```rust /// What a summary state computes, with every lifted selection removed. -pub struct SummaryStateDefinition { +pub struct AggregateDefinition { /// The group the unlifted operators read: the first input under the /// SummaryAgg or Aggregate that is not Filter / range TimeRange / /// TimeShift / direct-column Project. `None` when that input is a `Scan` with a @@ -256,13 +256,13 @@ pub struct SummaryStateDefinition { pub unlifted_operators: Vec>>, } -pub struct SummaryCoverage { - pub(crate) definition: SummaryStateDefinition, +pub struct AggregateCoverage { + pub(crate) definition: AggregateDefinition, /// Union of boxes over the output rows of the unlifted operators' input. pub(crate) selection: Vec, } -impl SummaryCoverage { +impl AggregateCoverage { pub(crate) fn derive_from_inputs( operator: &Operator>, ) -> Result; @@ -290,8 +290,8 @@ G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryA | 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input_group` | ```text -G4.aggregate_coverage = SummaryCoverage { - definition: SummaryStateDefinition { +G4.aggregate_coverage = AggregateCoverage { + definition: AggregateDefinition { input_group: Some(G1), unlifted_operators: [SummaryAgg[KLL]], }, @@ -304,8 +304,8 @@ G4.aggregate_coverage = SummaryCoverage { **Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: ```text -G10.aggregate_coverage = SummaryCoverage { - definition: SummaryStateDefinition { +G10.aggregate_coverage = AggregateCoverage { + definition: AggregateDefinition { input_group: Some(G1), unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], }, @@ -331,7 +331,7 @@ The `definition` is what remains of the operators the walk passed, with every co | a `Filter` whose conditions all move into `selection` | removed | | a `Filter` with some conditions that stay | kept, with only the conditions that stay; a scalar subquery among them names its group | | `SummaryAgg.filter` | trimmed the same way | -| the one range `TimeRange`, and every `TimeShift` (with or without `@`), when they become one interval on the timestamp column | removed | +| the one range `TimeRange`, and every `TimeShift` (an offset or `@ T`; not `@ start()` or `@ end()`, which stay unlifted), when they become one interval on the timestamp column | removed | | a `TimeShift` with no range `TimeRange`, two or more range `TimeRange`s, two `@`s, or an interval that does not combine with a timestamp condition already lifted | unchanged: no time is taken out | | a direct-column `Project` | unchanged | | a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input_group` is `None` | @@ -391,7 +391,7 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl | Operator it must pass | Can it pass? | Why | Example | |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | -| `TimeRange` (range) or `TimeShift` (with or without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | +| `TimeRange` (range) or `TimeShift` (an offset or `@ T`; `@ start()` and `@ end()` are not lifted) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | | `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | @@ -951,8 +951,8 @@ OperatorNode §6.1 │ (θ, `SketchParams`, is on the candidate) §6.4 └── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) -MemoGroup.group_properties.aggregate_coverage: SummaryCoverage §6.6 - └── definition: SummaryStateDefinition { input_group, unlifted_operators } +MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 + └── definition: AggregateDefinition { input_group, unlifted_operators } ``` | Subsection | Defines | Used by | @@ -962,15 +962,15 @@ MemoGroup.group_properties.aggregate_coverage: SummaryCoverage §6.6 | §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.summary_type` | | §6.4 Summary algorithms and parameters | `SketchAlgorithm`, `SketchParams`, `HydraParams`, exact/sample/wavelet/model kinds and params | the payload of each non-plain `FieldDataType` variant; θ on a candidate's `LogicalProperties` | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | -| §6.6 Summary coverage | `SummaryCoverage`, `SummaryStateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of summary-state memo groups | +| §6.6 Summary coverage | `AggregateCoverage`, `AggregateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of summary-state memo groups | -**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). +**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `AggregateCoverage` (§6.6). ### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) Every operator in a DAG is wrapped in an `OperatorNode`. `coverage()` is explained in §6.6. -Shown as of #646. On `main` today the node still has a declared `pub coverage: Option` field (with `with_coverage()` and `requires_coverage()`); #646 replaces it with the derived `coverage()` below. +Shown as of #646. On `main` today the node still has a declared `pub coverage: Option` field (with `with_coverage()` and `requires_coverage()`); #646 replaces it with the derived `coverage()` below. ```rust /// One node of the DAG. Immutable and shared through `Rc`. @@ -995,7 +995,7 @@ impl OperatorNode { /// The node's coverage (§4.2), derived on first use and then cached. /// `Some` for a `SummaryAgg` and a valid `SummaryMerge`; `None` for every /// other node. - pub fn coverage(&self) -> Option<&SummaryCoverage>; + pub fn coverage(&self) -> Option<&AggregateCoverage>; } ``` @@ -1373,15 +1373,15 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Summary coverage (`crates/types/src/ir/properties/summary_coverage.rs`) +### 6.6 Summary coverage (`crates/types/src/ir/properties/aggregate_coverage.rs`) -The code for §4. `SummaryStateDefinition` is shown in §4.2.2. +The code for §4. `AggregateDefinition` is shown in §4.2.2. ```rust -pub struct SummaryCoverage { +pub struct AggregateCoverage { /// What the state computes: its input group and the operators the walk /// did not lift (§4.2.2). - pub(crate) definition: SummaryStateDefinition, + pub(crate) definition: AggregateDefinition, /// Which rows went in: a union of boxes. A row is in the state if it is /// in at least one box. pub(crate) selection: Vec, @@ -1437,9 +1437,9 @@ A PromQL range `m[r]` evaluated at t reads the samples in (t − r, t]: left-ope | SQL `WHERE price > 10` | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | ```rust -impl SummaryCoverage { +impl AggregateCoverage { /// The coverage of a summary-state group computed by `operator` over its - /// input groups (§4.2.2). Errors: `NotSummary` for an operator that is + /// input groups (§4.2.2). Errors: `NoCoverage` for an operator that is /// not a `SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge` or a /// `Filter` keeping whole states; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` /// that does not expand to one build over its whole window; for a diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 4af0ad108..d57d603a4 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -27,7 +27,7 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- needs two timings; `split_shared_by_timing` copies it for one side first) or `IllegalChildDataState`. These are internal errors in a candidate the stages built, not bad user input. -- A summary build's `SummaryCoverage` (which rows and time range its state +- A summary build's `AggregateCoverage` (which rows and time range its state represents) is derived, never declared; a merge's inputs must be disjoint as its family's `MergeInputRule` requires, or coverage fails (`InputsOverlap`). From f68d414aa5f009f4b34b20654835d4d83b04ef23 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:08:34 +0000 Subject: [PATCH 07/12] refactor(types): third review of the memo: aggregate coverage with its derivation, one serialization per expression (#1207 PR 2 review) - D1: AggregateCoverage, AggregateDefinition and CoverageError move to stages/logical/aggregate_coverage.rs with the derivation, now the free function aggregate_coverage; ir/properties/selection.rs keeps the selection algebra (SelectionBox, Constraint, ColumnIdentity, IntervalConstraintLimit, constraint_of). - 1, 2: an expression is serialized once, with its group ids; the hash reads that string; a group is equal by typed equality, then the cached strings; a candidate by typed equality first. - 3: validate_expression_over_input_groups returns a merge's coverage, which derive_group_properties reuses. - 4: OutputColumnSource is crate-private; aggregate_output_columns deleted. - 5, 6: ranked_row_projection's doc restored; stale docs fixed. - 7: ValueSign and the sign derivation (column_signs) in column_properties.rs. - 9: MemoGroup's parents and queries are pub(super); Memo::queries(). - 10: the memo fixtures are in test_support::memo_fixtures, shared by the memo's unit tests and the integration tests. - 11: a measure's input column is computed once. - 12: Source::is_time_series. --- crates/devtools/src/bin/asap_planner.rs | 2 +- crates/devtools/src/planning_stages_text.rs | 2 +- crates/integration-tests/src/lib.rs | 2 +- crates/planner/src/lib.rs | 2 +- crates/stage0-memo/src/lib.rs | 14 +- .../src/computation_targets.rs | 16 +- crates/types/src/ir/operator/arguments.rs | 8 + crates/types/src/ir/operator/asap.rs | 4 +- crates/types/src/ir/operator/input.rs | 12 +- crates/types/src/ir/properties/mod.rs | 11 +- .../{aggregate_coverage.rs => selection.rs} | 117 +-------- .../types/src/ir/schema/aggregate_schema.rs | 31 +-- crates/types/src/ir/schema/error.rs | 4 +- crates/types/src/ir/schema/mod.rs | 2 +- crates/types/src/ir/schema/topk.rs | 7 +- crates/types/src/lib.rs | 2 +- crates/types/src/plan/materialized_view.rs | 9 +- crates/types/src/plan/physical_asap_dag.rs | 2 +- .../src/stages/logical/aggregate_coverage.rs | 246 +++++++++++++----- .../src/stages/logical/column_properties.rs | 23 +- crates/types/src/stages/logical/group.rs | 40 ++- .../src/stages/logical/memo/hash_consing.rs | 68 ++--- crates/types/src/stages/logical/memo/mod.rs | 29 ++- crates/types/src/stages/logical/memo/tests.rs | 178 ++++++++----- crates/types/src/stages/logical/mod.rs | 4 +- crates/types/src/stages/mod.rs | 9 +- crates/types/src/test_support.rs | 4 +- .../test_support}/memo_fixtures.rs | 17 +- crates/types/tests/main.rs | 1 - crates/types/tests/memo_aggregate_coverage.rs | 9 +- crates/types/tests/memo_group_properties.rs | 41 ++- .../proposals/asap-primitive-schema.md | 4 +- 32 files changed, 479 insertions(+), 441 deletions(-) rename crates/types/src/ir/properties/{aggregate_coverage.rs => selection.rs} (81%) rename crates/types/{tests => src/test_support}/memo_fixtures.rs (78%) diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index bef383ab9..a802820f0 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -757,7 +757,7 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) "children": group.expression.children().into_iter().map(|g| g.0).collect::>(), "parents": ids(group.parent_groups()), "queries": group.reached_by_queries(), - "roots_of": queries_rooted_at(&memo.queries, MemoGroupId(id as u32)), + "roots_of": queries_rooted_at(memo.queries(), MemoGroupId(id as u32)), "node": node, "candidate_count": candidates.len(), "candidate_count_by_origin": by_origin, diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index d3e422865..160d792db 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -201,7 +201,7 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi todo!("#1207 PR 9: print each candidate's expression, logical properties and input requirements") } let roots: Vec = memo - .queries + .queries() .iter() .map(|query| match &query.root { QueryRoot::Operator(group) => format!("G{}", group.0), diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index a240bcb7c..c933d4857 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -302,7 +302,7 @@ pub mod planning { .iter() .enumerate() .filter(|(group, _)| { - memo.queries[q] + memo.queries()[q] .root .operator_refs() .contains(&&asap_types::stages::MemoGroupId(*group as u32)) diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index 21e6cf6a5..c88e260f7 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -57,7 +57,7 @@ pub use asap_stage3_plan_selection::{ AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, }; pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; -pub use asap_types::ir::properties::aggregate_coverage::{ +pub use asap_types::ir::properties::selection::{ ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, }; pub use asap_types::ir::properties::ExecutionTiming; diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index de5a9bed5..8a7b78ddf 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -165,7 +165,7 @@ mod tests { /// Query `query`'s root group. fn root(memo: &Memo, query: usize) -> MemoGroupId { - match &memo.queries[query].root { + match &memo.queries()[query].root { QueryRoot::Operator(group) => *group, QueryRoot::Scalar(_) => panic!("expected an operator root"), } @@ -247,8 +247,8 @@ mod tests { let memo = memo_of_queries(&roots, &targets, &DataWorkload::default()).unwrap(); assert_eq!(root(&memo, 0), root(&memo, 1)); assert_eq!(memo.groups().len(), 2); - assert_eq!(memo.queries[0].accuracy_target, targets[0]); - assert_eq!(memo.queries[1].accuracy_target, targets[1]); + assert_eq!(memo.queries()[0].accuracy_target, targets[0]); + assert_eq!(memo.queries()[1].accuracy_target, targets[1]); } /// `vector(scalar(sum by (service) (m)))`. @@ -282,7 +282,7 @@ mod tests { assert_eq!(memo.groups().len(), 3); assert_eq!(root(&memo, 0), root(&memo, 1)); let sum = MemoGroupId(1); - assert_eq!(memo.queries[2].root.operator_refs(), vec![&sum]); + assert_eq!(memo.queries()[2].root.operator_refs(), vec![&sum]); assert_eq!( memo.groups()[sum.0 as usize].reached_by_queries(), &[0, 1, 2] @@ -368,16 +368,16 @@ mod tests { }; let memo = memo_of_queries(&roots, &[None, None], &declared).unwrap(); assert_eq!( - memo.queries[0].number_of_values_per_query_evaluation, + memo.queries()[0].number_of_values_per_query_evaluation, Some(1000) ); assert_eq!( - memo.queries[1].number_of_values_per_query_evaluation, + memo.queries()[1].number_of_values_per_query_evaluation, Some(1) ); let undeclared = memo_of_roots(&roots).unwrap(); assert_eq!( - undeclared.queries[0].number_of_values_per_query_evaluation, + undeclared.queries()[0].number_of_values_per_query_evaluation, None ); } diff --git a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs index 1cf6f0258..f46d15f25 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs @@ -42,8 +42,7 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use asap_types::ir::operator::arguments::Reduction; -use asap_types::ir::operator::input::scans_only_time_series; +use asap_types::ir::operator::arguments::{Reduction, Source}; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; use asap_types::ir::schema::FieldDataType; @@ -349,7 +348,7 @@ pub fn generate_per_computation_candidates( // a group several queries read is one node. let node = |g: &MemoGroupId| Rc::clone(&logical_nodes[g.0 as usize]); let roots: Vec = memo - .queries + .queries() .iter() .map(|query| match &query.root { QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), @@ -738,11 +737,12 @@ fn prune_topk_heaps_over_one_row_per_item( // A PromQL top-k ranks an instant vector: one row per series or // group. (A SQL top-k keeps its heaps.) The aggregate scans what its // input does. - if !scans_only_time_series( - &memo.groups()[target.aggregate_group.0 as usize] - .group_properties - .scanned_sources, - ) { + if !memo.groups()[target.aggregate_group.0 as usize] + .group_properties + .scanned_sources + .iter() + .all(Source::is_time_series) + { continue; } prune_own(&mut target.ways_to_compute, |candidate| { diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index fe0533ffc..2b6ff8f78 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -178,6 +178,14 @@ pub enum Source { Table { table_ref: String }, } +impl Source { + /// Whether it is a time series (PromQL rows): a PromQL top-k over rows + /// that scan only time series ranks one row per series. + pub fn is_time_series(&self) -> bool { + matches!(self, Self::TimeSeries { .. }) + } +} + /// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's /// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison /// operator has exactly one representation (and one `Display`) across the IR. diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index f24a426b6..037c6d57f 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -12,8 +12,8 @@ use super::non_asap::NonASAPOp; use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::arguments::Reduction; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; -use crate::ir::schema::aggregate_schema::aggregate_output; -use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, OutputColumnSource, Schema}; +use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; use crate::summary_type::SummaryTypeCapability; use crate::summary_type::{SketchStatistic, SummaryUpdate}; diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index a489a1047..2aa3b9755 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -10,9 +10,9 @@ use std::rc::Rc; use super::arguments::Source; use super::node::{Operator, OperatorNode, OperatorResultKind}; use super::non_asap::NonASAPOp; -use crate::ir::properties::aggregate_coverage::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; +use crate::stages::AggregateCoverage; /// An input of an operator, as the rules read it. pub trait OperatorInput: Sized + Clone { @@ -42,15 +42,7 @@ pub(crate) fn sources_scanned_by(operator: &Operator) -> BT } } -/// Whether every source in `sources` is a time series (PromQL rows): a -/// PromQL top-k ranks one row per series. -pub fn scans_only_time_series(sources: &BTreeSet) -> bool { - sources - .iter() - .all(|source| matches!(source, Source::TimeSeries { .. })) -} - -/// A front-end node tree. Node trees carry no summary coverage. +/// A front-end node tree. Node trees carry no aggregate coverage. impl OperatorInput for Rc { fn schema(&self) -> &Schema { &self.schema diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index b491c013a..380b0bd8b 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -1,11 +1,14 @@ //! #511 §2.2–2.3: the planning properties a node carries besides its -//! operator and schema. +//! operator and schema (when its value is produced), and the selection +//! algebra an aggregate's coverage (`stages::AggregateCoverage`) is stated +//! in. //! //! - [`execution_data_state`] — when a node's value is produced //! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). -//! - [`aggregate_coverage`] — what an aggregate (a summary state or an exact -//! `Aggregate`) covers. -pub mod aggregate_coverage; +//! - [`selection`] — which rows of a computation an aggregate's state took: +//! boxes of per-column constraints, the selection algebra aggregate +//! coverage (`stages::AggregateCoverage`) is built from. pub mod execution_data_state; +pub mod selection; pub use execution_data_state::{DataPrimitive, ExecutionDataState, ExecutionTiming}; diff --git a/crates/types/src/ir/properties/aggregate_coverage.rs b/crates/types/src/ir/properties/selection.rs similarity index 81% rename from crates/types/src/ir/properties/aggregate_coverage.rs rename to crates/types/src/ir/properties/selection.rs index aec4f7815..5456a0069 100644 --- a/crates/types/src/ir/properties/aggregate_coverage.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -1,62 +1,19 @@ -//! Aggregate coverage: what an aggregate (a summary state or an exact -//! `Aggregate`) covers: `definition` (what it computes) and `selection` (which -//! output rows of that computation it took). Design: -//! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after -//! Goldstein & Larson's view matching. -//! -//! Coverage is derived once for each aggregate memo group, from its input -//! groups (`stages::logical::aggregate_coverage`); node trees carry none. +//! The selection algebra of aggregate coverage +//! (`stages::AggregateCoverage::selection`): which output rows of a +//! computation a state took, as a union of boxes ([`SelectionBox`]), each a +//! [`Constraint`] per column ([`ColumnIdentity`]): value sets and intervals, +//! with interval limits relative to the evaluation time +//! ([`IntervalConstraintLimit`]). A predicate conjunct becomes a constraint +//! through `constraint_of`. use std::cmp::Ordering; use std::collections::BTreeMap; use std::ops::Bound; use serde::{Deserialize, Serialize}; -use thiserror::Error; -use crate::ir::operator::node::Operator; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::ScalarExpr; use crate::ir::schema::ColumnId; -use crate::stages::MemoGroupId; -use crate::summary_type::MergeInputRule; - -/// What an aggregate computes, with every lifted selection removed. -#[derive(Debug, Clone, PartialEq)] -pub struct AggregateDefinition { - /// The group the unlifted operators read: the first input under the - /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ - /// / direct-column Project. `None` when that input is a `Scan` with a - /// liftable predicate (a PromQL label matcher): it is lifted too, and the - /// `Scan` with the rest of its predicates is the last unlifted operator, - /// so states over one source with different matchers may merge. - pub input_group: Option, - /// The operators not lifted into the selection, listed top down from the - /// SummaryAgg, each with only its residual arguments. `None` marks an - /// operator's main input (the next unlifted operator, or input_group); - /// `Some(g)` is another input group it reads, such as the group a scalar - /// subquery in a Filter reads. Two states may merge only if input_group - /// and these operators are equal. - pub unlifted_operators: Vec>>, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct AggregateCoverage { - pub(crate) definition: AggregateDefinition, - /// Union of boxes over the output rows of the unlifted operators' input. - pub(crate) selection: Vec, -} - -#[cfg(any(test, feature = "test-support"))] -impl AggregateCoverage { - pub fn definition(&self) -> &AggregateDefinition { - &self.definition - } - - /// Union of boxes over the output rows of the unlifted operators' input. - pub fn selection(&self) -> &[SelectionBox] { - &self.selection - } -} /// A conjunction of per-column constraints. The timestamp column is one of /// the columns; there is no separate time field. A column it does not @@ -221,66 +178,6 @@ mod limit_bound { } } -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum CoverageError { - /// The node has no coverage: it is no aggregate (`SummaryAgg`, - /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a - /// `TimeRange`/`TimeShift` over a chain whose windows are not one - /// interval on the timestamp column. A caller bug, or bad input when it - /// is a merge's input. - #[error("coverage is derived only for aggregates and one time interval over a chain")] - NoCoverage, - /// A `SummaryMerge` without inputs: bad input. - #[error("summary merge requires at least one input")] - EmptyMerge, - /// The inputs of a `SummaryMerge` compute different things: bad input. - #[error("summary merge inputs compute different things")] - DefinitionMismatch, - /// The inputs of a `SummaryMerge` are not proven to relate as the - /// family's merge requires (disjoint, or disjoint in time for counter - /// states): bad input. - #[error( - "summary merge inputs are not proven {}, which the family's merge requires", - match required { - MergeInputRule::DisjointInTime => "disjoint in time", - _ => "disjoint", - } - )] - InputsOverlap { required: MergeInputRule }, - /// A regrouping `SummaryMerge` key that is not one of the group keys - /// its input states are built by (a build grouped `without` keys has - /// none): bad input. - #[error("regrouping merge key {key} is not a group key of its input states")] - NotAGroupKey { key: ColumnId }, - /// A regrouping `SummaryMerge` of states whose summary type does not - /// merge different groups: counter states (`Rate`, `Increase`) merge - /// consecutive windows of one series, a summary type without a merge - /// merges nothing, and a Hydra's groups share one state. Bad input. - #[error("the inputs' summary type cannot merge different groups")] - SummaryTypeCannotRegroup, - /// The build a regrouping `SummaryMerge` stands for does not derive: - /// bad input. - #[error("the regrouped build does not derive: {reason}")] - RegroupedBuild { reason: String }, -} - -impl AggregateCoverage { - /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, - /// of its input group and unlifted operators. Two equal definitions - /// (the equality a merge requires) of one memo get equal ids, so a - /// summary store can key states by this id and a concrete selection. - pub(crate) fn definition_id(&self) -> String { - const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; - const PRIME: u128 = 0x0000000001000000000000000000013b; - let mut hash = OFFSET; - for byte in format!("{:?}", self.definition).bytes() { - hash ^= u128::from(byte); - hash = hash.wrapping_mul(PRIME); - } - format!("{hash:032x}") - } -} - /// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. pub(crate) fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { diff --git a/crates/types/src/ir/schema/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs index 240d1863a..5fe7edd11 100644 --- a/crates/types/src/ir/schema/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -18,7 +18,7 @@ use crate::ir::SchemaDerivationError; /// `[InputColumn(job), Measure { index: 0, input_column: Some(value) }]`. The one layout the output schema /// ([`aggregate_output_schema`]) and the group properties read. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum OutputColumnSource { +pub(crate) enum OutputColumnSource { InputColumn(ColumnId), Measure { index: usize, @@ -100,17 +100,8 @@ pub fn aggregate_output_schema( Ok(aggregate_output(in_schema, reduction, measures, output_names)?.0) } -/// Where each column of [`aggregate_output_schema`]'s output comes from, in -/// its order. -pub fn aggregate_output_columns( - in_schema: &Schema, - reduction: &Reduction, - measures: &[AggIntent], -) -> Result, SchemaDerivationError> { - Ok(aggregate_output(in_schema, reduction, measures, &[])?.1) -} - -/// [`aggregate_output_schema`] and [`aggregate_output_columns`] in one pass. +/// [`aggregate_output_schema`] and where each of its columns comes from +/// ([`OutputColumnSource`]), in one pass. pub(crate) fn aggregate_output( in_schema: &Schema, reduction: &Reduction, @@ -200,8 +191,8 @@ pub(crate) fn aggregate_output( // Only the output *type* is read from here, so the leading column is // enough for the multi-column intents: `Cardinality` and `PearsonCorr` // both have a fixed output type that ignores it. - let in_col = - measure_input_column(in_schema, intent).map_or(&probe, |id| &in_schema.fields[id]); + let input_column = measure_input_column(in_schema, intent); + let in_col = input_column.map_or(&probe, |id| &in_schema.fields[id]); let mut out = intent.output_column(in_col); // A global extremum emits NULL for an empty input, even if its input // column is non-nullable. Grouped extrema only emit existing groups. @@ -221,7 +212,7 @@ pub(crate) fn aggregate_output( out_cols.push(out); sources.push(OutputColumnSource::Measure { index: i, - input_column: measure_input_column(in_schema, intent).or(value_col_idx), + input_column: input_column.or(value_col_idx), }); } // `count_values` groups by (by-keys ∪ the synthesized value label), so the @@ -303,8 +294,8 @@ fn without_output_schema( // Only the output *type* is read from here, so the leading column is // enough for the multi-column intents: `Cardinality` and `PearsonCorr` // both have a fixed output type that ignores it. - let in_col = - measure_input_column(in_schema, intent).map_or(&probe, |id| &in_schema.fields[id]); + let input_column = measure_input_column(in_schema, intent); + let in_col = input_column.map_or(&probe, |id| &in_schema.fields[id]); let mut out = intent.output_column(in_col); if let Some((arg, _)) = intent .arg_selector_columns(in_schema) @@ -319,8 +310,7 @@ fn without_output_schema( out_cols.push(out); sources.push(OutputColumnSource::Measure { index: i, - input_column: measure_input_column(in_schema, intent) - .or_else(|| in_schema.column_id("value")), + input_column: input_column.or_else(|| in_schema.column_id("value")), }); } Ok(( @@ -357,8 +347,7 @@ mod tests { /// Each output column's source, and one source per column of /// `aggregate_output_schema`'s output. fn columns(reduction: Reduction, measures: &[AggIntent]) -> Vec { - let sources = aggregate_output_columns(&series(), &reduction, measures).unwrap(); - let schema = aggregate_output_schema(&series(), &reduction, measures, &[]).unwrap(); + let (schema, sources) = aggregate_output(&series(), &reduction, measures, &[]).unwrap(); assert_eq!(sources.len(), schema.fields.len()); sources } diff --git a/crates/types/src/ir/schema/error.rs b/crates/types/src/ir/schema/error.rs index 4aa37e4ce..f10cf38d5 100644 --- a/crates/types/src/ir/schema/error.rs +++ b/crates/types/src/ir/schema/error.rs @@ -16,6 +16,6 @@ pub enum SchemaDerivationError { EmptyConcat, #[error("invalid per-series sample column: {0}")] InvalidSampleColumn(String), - #[error("invalid summary coverage: {0}")] - Coverage(#[from] crate::ir::properties::aggregate_coverage::CoverageError), + #[error("invalid aggregate coverage: {0}")] + Coverage(#[from] crate::stages::CoverageError), } diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 8d5981154..72e3e5627 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -18,7 +18,7 @@ pub mod error; pub mod series_identity; pub mod topk; -pub use aggregate_schema::{aggregate_output_columns, aggregate_output_schema, OutputColumnSource}; +pub use aggregate_schema::aggregate_output_schema; use serde::{Deserialize, Serialize}; diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 9436c94bf..3ad853e70 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,5 +1,6 @@ //! Ranked-row output contract shared by logical TopK and its implementations. -use crate::ir::operator::input::{scans_only_time_series, OperatorInput}; +use crate::ir::operator::arguments::Source; +use crate::ir::operator::input::OperatorInput; use crate::ir::operator::Reduction; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; @@ -55,6 +56,8 @@ pub fn ranked_column(child: &I) -> Result( child: &I, reduction: &Reduction, @@ -83,7 +86,7 @@ pub(crate) fn ranked_row_projection( alias: Some(output_name.into()), expr: ScalarExpr::Column(value), }); - let promql = scans_only_time_series(&child.scanned_sources()); + let promql = child.scanned_sources().iter().all(Source::is_time_series); for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index c33162e00..6c2659b09 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -33,6 +33,6 @@ pub mod plan; pub mod serde_f64; pub mod stages; pub mod summary_type; -#[cfg(feature = "test-support")] +#[cfg(any(test, feature = "test-support"))] pub mod test_support; pub mod workload; diff --git a/crates/types/src/plan/materialized_view.rs b/crates/types/src/plan/materialized_view.rs index 3c8f49484..27ff1a3af 100644 --- a/crates/types/src/plan/materialized_view.rs +++ b/crates/types/src/plan/materialized_view.rs @@ -76,9 +76,10 @@ use crate::ir::operator::arguments::Source; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::{Operator, OperatorNode}; use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::properties::aggregate_coverage::{AggregateCoverage, CoverageError, SelectionBox}; use crate::ir::properties::execution_data_state::ExecutionTiming; +use crate::ir::properties::selection::SelectionBox; use crate::stages::Lifetime; +use crate::stages::{AggregateCoverage, CoverageError}; /// One materialized view of a DAG: what its stored state is, what it is /// refreshed from, when, and for how long it is retained. @@ -89,7 +90,7 @@ pub struct MaterializedView { pub node_id: PhysicalASAPNodeId, /// Which run refreshes it. pub(crate) refresh: MaterializedViewRefresh, - /// What the stored state is: the node's summary coverage. + /// What the stored state is: the node's aggregate coverage. pub(crate) coverage: ViewCoverage, /// What its refresh reads. pub inputs: MaterializedViewInputs, @@ -119,7 +120,7 @@ impl MaterializedView { } } -/// A view's summary coverage +/// A view's aggregate coverage /// ([`AggregateCoverage`], `docs/design_docs/proposals/asap-primitive-schema.md` /// §4.2): what it computes and which rows went in. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -349,7 +350,7 @@ pub enum InstallableDagError { ViewReadsUnwindowedSource { node_id: PhysicalASAPNodeId }, /// Unsatisfiable request: view `node_id` is not a summary state with a /// coverage (`reason`), so a runtime cannot say what it stores. - #[error("materialized view {node_id} has no summary coverage: {reason}")] + #[error("materialized view {node_id} has no aggregate coverage: {reason}")] ViewWithoutCoverage { node_id: PhysicalASAPNodeId, reason: String, diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index b6b126da7..e4ff4b4fd 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -258,7 +258,7 @@ impl PhysicalASAPDAG { /// Every node of this DAG as IR, by node id, with its exported /// guarantee (an operator node has no physical properties): what a caller derives /// IR properties of a plan file's node from, such as its - /// [`AggregateCoverage`](crate::ir::properties::aggregate_coverage::AggregateCoverage). + /// [`AggregateCoverage`](crate::stages::AggregateCoverage). /// Shared inputs stay shared. Validates the DAG first. pub(crate) fn operator_nodes( &self, diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index 57792c18a..fd1cd19cc 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -1,30 +1,35 @@ -//! Aggregate coverage, a group property: [`AggregateCoverage`] derived once -//! for each aggregate memo group from its input groups. +//! Aggregate coverage, a group property: what an aggregate (a summary state +//! or an exact `Aggregate`) covers ([`AggregateCoverage`]): `definition` +//! (what it computes) and `selection` (which output rows of that computation +//! it took, in the IR's selection algebra, `ir::properties::selection`). +//! Design: `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after +//! Goldstein & Larson's view matching. //! -//! It is derived for a summary state or an exact `Aggregate`, never -//! declared, and -//! stored in its `GroupProperties::aggregate_coverage`. Walking down from a -//! `SummaryAgg` or `Aggregate` through the input groups' expressions, a -//! predicate conjunct moves into `selection` when it can be lifted to the -//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and -//! direct-column `Project` items) and it is a value set or an interval on -//! one column. The `TimeRange` and `TimeShift`s become one interval on the -//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in -//! `definition` as a residual, so two states are merged only when they -//! compute the same thing over disjoint rows. Node trees carry no coverage. +//! It is derived once for each aggregate memo group from its input groups +//! ([`aggregate_coverage`]), never declared, and stored in its +//! `GroupProperties::aggregate_coverage`. Walking down from a `SummaryAgg` +//! or `Aggregate` through the input groups' expressions, a predicate +//! conjunct moves into `selection` when it can be lifted to the top (through +//! `Filter`, a range `TimeRange`, a `TimeShift` and direct-column `Project` +//! items) and it is a value set or an interval on one column. The +//! `TimeRange` and `TimeShift`s become one interval on the timestamp column +//! ([`IntervalConstraintLimit`]). Everything else stays in `definition` as a +//! residual, so two states are merged only when they compute the same thing +//! over disjoint rows. Node trees carry no coverage. use std::collections::BTreeMap; use std::ops::Bound; +use thiserror::Error; + use super::MemoGroupAsInput; use crate::ir::operator::arguments::{AtModifier, Reduction}; use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::properties::aggregate_coverage::{ - constraint_of, union, AggregateCoverage, AggregateDefinition, ColumnIdentity, Constraint, - CoverageError, IntervalConstraintLimit, SelectionBox, +use crate::ir::properties::selection::{ + constraint_of, union, ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, }; use crate::ir::scalar::expr_ir::ScalarValue; use crate::ir::scalar::{Predicate, ScalarExpr}; @@ -33,65 +38,162 @@ use crate::ir::SchemaDerivationError; use crate::stages::MemoGroupId; use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; +/// What an aggregate computes, with every lifted selection removed. +#[derive(Debug, Clone, PartialEq)] +pub struct AggregateDefinition { + /// The group the unlifted operators read: the first input under the + /// `SummaryAgg` or exact `Aggregate` that is not a `Filter`, a range + /// `TimeRange`, a `TimeShift` without `@ start()`/`@ end()` or a + /// direct-column `Project`. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator, + /// so states over one source with different matchers may merge. + pub input_group: Option, + /// The operators not lifted into the selection, listed top down from the + /// `SummaryAgg` or `Aggregate`, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or input_group); + /// `Some(g)` is another input group it reads, such as the group a scalar + /// subquery in a Filter reads. Two states may merge only if input_group + /// and these operators are equal. + pub unlifted_operators: Vec>>, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AggregateCoverage { + pub(crate) definition: AggregateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. + pub(crate) selection: Vec, +} + +#[cfg(any(test, feature = "test-support"))] impl AggregateCoverage { - /// Coverage of what a memo group computes with `operator` over its - /// input groups: a `SummaryAgg`, an exact `Aggregate`, a - /// `SummaryTumblingWindowChain` (a build over its input, the timestamp - /// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the - /// chain's coverage with the windows' interval on the timestamp column), - /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped - /// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` - /// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge - /// reads its inputs' stored coverage and fails - /// unless every input has the same definition and their selections are - /// pairwise disjoint; a family whose merge is idempotent - /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may - /// also merge overlapping selections, and a counter family - /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. - pub(super) fn derive_from_inputs<'m>( - operator: &Operator>, - ) -> Result { - let as_built = |op: &ASAPOp>| -> ASAPOp> { - op.map_children(|input| InputOrTemporaryBuild::Input(*input)) - }; - match operator { - Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { - let top = - unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); - of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) - } - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - if operator.output_kind() == OperatorResultKind::State => - { - of_window_selection(operator) - } - Operator::NonASAP(_) => match selected_build(operator)? { - Some(build) => of_summary_agg(&build), - None => Err(CoverageError::NoCoverage), - }, - Operator::ASAP( - op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), - ) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryMerge { - children, group_by, .. - }) => { - let inputs = children - .iter() - .map(|child| { - child - .aggregate_coverage() - .cloned() - .ok_or(CoverageError::NoCoverage) - }) - .collect::, _>>()?; - let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; - match group_by { - None => Ok(merged), - Some(keys) => regrouped(merged, &children[0], keys), - } + pub fn definition(&self) -> &AggregateDefinition { + &self.definition + } + + /// Union of boxes over the output rows of the unlifted operators' input. + pub fn selection(&self) -> &[SelectionBox] { + &self.selection + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CoverageError { + /// The node has no coverage: it is no aggregate (`SummaryAgg`, + /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a + /// `TimeRange`/`TimeShift` over a chain whose windows are not one + /// interval on the timestamp column. A caller bug, or bad input when it + /// is a merge's input. + #[error("coverage is derived only for aggregates and one time interval over a chain")] + NoCoverage, + /// A `SummaryMerge` without inputs: bad input. + #[error("summary merge requires at least one input")] + EmptyMerge, + /// The inputs of a `SummaryMerge` compute different things: bad input. + #[error("summary merge inputs compute different things")] + DefinitionMismatch, + /// The inputs of a `SummaryMerge` are not proven to relate as the + /// family's merge requires (disjoint, or disjoint in time for counter + /// states): bad input. + #[error( + "summary merge inputs are not proven {}, which the family's merge requires", + match required { + MergeInputRule::DisjointInTime => "disjoint in time", + _ => "disjoint", + } + )] + InputsOverlap { required: MergeInputRule }, + /// A regrouping `SummaryMerge` key that is not one of the group keys + /// its input states are built by (a build grouped `without` keys has + /// none): bad input. + #[error("regrouping merge key {key} is not a group key of its input states")] + NotAGroupKey { key: ColumnId }, + /// A regrouping `SummaryMerge` of states whose summary type does not + /// merge different groups: counter states (`Rate`, `Increase`) merge + /// consecutive windows of one series, a summary type without a merge + /// merges nothing, and a Hydra's groups share one state. Bad input. + #[error("the inputs' summary type cannot merge different groups")] + SummaryTypeCannotRegroup, + /// The build a regrouping `SummaryMerge` stands for does not derive: + /// bad input. + #[error("the regrouped build does not derive: {reason}")] + RegroupedBuild { reason: String }, +} + +impl AggregateCoverage { + /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, + /// of its input group and unlifted operators. Two equal definitions + /// (the equality a merge requires) of one memo get equal ids, so a + /// summary store can key states by this id and a concrete selection. + pub(crate) fn definition_id(&self) -> String { + const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; + const PRIME: u128 = 0x0000000001000000000000000000013b; + let mut hash = OFFSET; + for byte in format!("{:?}", self.definition).bytes() { + hash ^= u128::from(byte); + hash = hash.wrapping_mul(PRIME); + } + format!("{hash:032x}") + } +} + +/// Coverage of what a memo group computes with `operator` over its +/// input groups: a `SummaryAgg`, an exact `Aggregate`, a +/// `SummaryTumblingWindowChain` (a build over its input, the timestamp +/// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the +/// chain's coverage with the windows' interval on the timestamp column), +/// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped +/// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` +/// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge +/// reads its inputs' stored coverage and fails +/// unless every input has the same definition and their selections are +/// pairwise disjoint; a family whose merge is idempotent +/// ([`MergeInputRule::MayOverlap`] in the family-capability table) may +/// also merge overlapping selections, and a counter family +/// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. +pub(super) fn aggregate_coverage<'m>( + operator: &Operator>, +) -> Result { + let as_built = |op: &ASAPOp>| -> ASAPOp> { + op.map_children(|input| InputOrTemporaryBuild::Input(*input)) + }; + match operator { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = + unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); + of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if operator.output_kind() == OperatorResultKind::State => + { + of_window_selection(operator) + } + Operator::NonASAP(_) => match selected_build(operator)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NoCoverage), + }, + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => of_summary_agg(&as_built(op)), + Operator::ASAP(ASAPOp::SummaryMerge { + children, group_by, .. + }) => { + let inputs = children + .iter() + .map(|child| { + child + .aggregate_coverage() + .cloned() + .ok_or(CoverageError::NoCoverage) + }) + .collect::, _>>()?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; + match group_by { + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), } - Operator::ASAP(_) => Err(CoverageError::NoCoverage), } + Operator::ASAP(_) => Err(CoverageError::NoCoverage), } } diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index 0b2b6d1a7..ee5deaeff 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -5,26 +5,39 @@ use std::collections::BTreeMap; -use super::{MemoGroupAsInput, ValueSign}; +use super::MemoGroupAsInput; use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction}; use crate::ir::operator::asap::{summary_agg_layout, summary_build}; use crate::ir::operator::input::OperatorInput; use crate::ir::operator::AggIntent; use crate::ir::scalar::expr_ir::ScalarValue; use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; -use crate::ir::schema::{aggregate_output_columns, FieldDataType, OutputColumnSource, Schema}; +use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; +use crate::ir::schema::{FieldDataType, Schema}; use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; use crate::workload::bounds::source_name; use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; +/// The sign of one column's values, a group property derived from the +/// input groups like lineage (concepts doc, section 3, "Value sign"); a +/// `Scan`'s are declared per column in the data workload's source facts. +/// Error contracts that hold only for values of one sign read it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValueSign { + NonNegative, + NonPositive, + Unknown, +} + use ValueSign::{NonNegative, NonPositive, Unknown}; type Input<'m> = MemoGroupAsInput<'m>; /// The sign of each column of `output`, the output schema of `operator`, /// from its input groups' `column_signs` and, for a `Scan`, `source_facts`. -pub(super) fn output_column_sign( +pub(super) fn column_signs( operator: &Operator>, output: &Schema, source_facts: &BTreeMap, @@ -277,7 +290,7 @@ fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { } /// An aggregate's, by where each output column comes from -/// ([`aggregate_output_columns`]): a kept input column keeps its sign, a +/// ([`aggregate_output`]): a kept input column keeps its sign, a /// measure's value has [`measure_sign`] of its input column's, and a label /// a measure adds is `Unknown`. fn aggregate_signs( @@ -286,7 +299,7 @@ fn aggregate_signs( measures: &[AggIntent], output: &Schema, ) -> Vec { - let Ok(sources) = aggregate_output_columns(child.schema(), reduction, measures) else { + let Ok((_, sources)) = aggregate_output(child.schema(), reduction, measures, &[]) else { return vec![Unknown; output.fields.len()]; }; signs_from_sources(child, &sources, |index, input| { diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 168f07040..1d1765962 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -3,10 +3,10 @@ use std::collections::BTreeSet; -use super::column_properties::output_column_sign; +use super::aggregate_coverage::{aggregate_coverage, AggregateCoverage}; +use super::column_properties::{column_signs, ValueSign}; use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; use crate::ir::operator::input::{sources_scanned_by, OperatorInput}; -use crate::ir::properties::aggregate_coverage::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; @@ -33,12 +33,12 @@ pub struct MemoGroup { /// expression or a candidate's: kept up to date by /// [`Memo::find_or_add_group`](super::Memo::find_or_add_group) and /// [`Memo::add_candidate`](super::Memo::add_candidate). - pub parents: Vec, + pub(super) parents: Vec, /// The queries whose roots reach it ([`Self::reached_by_queries`]), /// ascending: kept up to date by /// [`Memo::add_query`](super::Memo::add_query) and /// [`Memo::add_candidate`](super::Memo::add_candidate). - pub queries: Vec, + pub(super) queries: Vec, /// Its candidates: empty after Stage 0, filled by Stage 1. pub candidates: Vec, } @@ -84,32 +84,22 @@ pub struct GroupProperties { pub scanned_sources: BTreeSet, } -/// The sign of one column's values, a group property derived from the -/// input groups like lineage (concepts doc, section 3, "Value sign"); a -/// `Scan`'s are declared per column in the data workload's source facts. -/// Error contracts that hold only for values of one sign read it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ValueSign { - NonNegative, - NonPositive, - Unknown, -} - /// Whether `operator` is a valid expression over its input groups /// ([`MemoGroupAsInput`]), never a node tree: its output schema derives, its /// inputs pass validation, and a merge reads only states it may merge. -/// Returns the output schema. +/// Returns the output schema and, for a merge, the coverage its check +/// derived. pub(super) fn validate_expression_over_input_groups( operator: &Operator>, -) -> Result { +) -> Result<(Schema, Option), SchemaDerivationError> { let schema = operator.output_schema()?; operator.validate_inputs()?; // A merge is valid only over states of one definition it may merge. - if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) { - AggregateCoverage::derive_from_inputs(operator)?; - } - Ok(schema) + let merge_coverage = match operator { + Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(aggregate_coverage(operator)?), + _ => None, + }; + Ok((schema, merge_coverage)) } /// The properties of a group computing `expression` over its input @@ -121,10 +111,10 @@ pub(super) fn derive_group_properties( expression: &Operator, ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); - let schema = validate_expression_over_input_groups(&operator)?; - let aggregate_coverage = AggregateCoverage::derive_from_inputs(&operator).ok(); + let (schema, merge_coverage) = validate_expression_over_input_groups(&operator)?; + let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(&operator).ok()); Ok(GroupProperties { - column_signs: output_column_sign(&operator, &schema, &memo.source_facts), + column_signs: column_signs(&operator, &schema, &memo.source_facts), depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), diff --git a/crates/types/src/stages/logical/memo/hash_consing.rs b/crates/types/src/stages/logical/memo/hash_consing.rs index 78c68be6f..61c489187 100644 --- a/crates/types/src/stages/logical/memo/hash_consing.rs +++ b/crates/types/src/stages/logical/memo/hash_consing.rs @@ -1,6 +1,8 @@ //! The one equality that makes two expressions one memo group, and two -//! candidates of a group one candidate: [`same_serialized_expression`] (operator, -//! arguments and input groups). [`Memo::find_or_add_group`] looks a group up +//! candidates of a group one candidate: typed equality of the operator, its +//! arguments and input groups, confirmed by equal serializations +//! ([`same_value`]; [`same_serialized_expression`] with each side's +//! serialization computed once). [`Memo::find_or_add_group`] looks a group up //! by it, Stage 0 included, and [`Memo::add_candidate`] finds a duplicate by //! it. Parameters θ are not an argument of any operator (they are a //! candidate's `LogicalProperties`), so builds of one family over one input @@ -16,52 +18,32 @@ use super::MemoGroupId; use crate::ir::Operator; use crate::stages::CandidateForMemoGroup; -/// `expression` with every input group replaced by `()`: the operator and -/// its arguments. -fn operator_and_arguments(expression: &Operator) -> Operator<()> { - expression.map_children(|_| ()) +/// The canonical JSON of `expression` (operator, arguments and input +/// groups), serialized once per expression: what [`expression_hash`] hashes +/// and [`same_serialized_expression`] compares. +pub(super) fn serialized_expression(expression: &Operator) -> String { + serde_json::to_string(expression).expect("an expression over memo groups serializes") } -/// The canonical JSON of `expression`'s operator and arguments, serialized -/// once per expression: what [`expression_hash`] hashes and -/// [`same_serialized_expression`] compares; `None` when it does not -/// serialize. -pub(super) fn serialized_operator(expression: &Operator) -> Option { - serde_json::to_string(&operator_and_arguments(expression)).ok() -} - -/// A hash consistent with [`same_serialized_expression`], used only to narrow the -/// groups compared: SipHash over `serialized`, the expression's -/// [`serialized_operator`], then the input groups. A non-finite `f64` -/// serializes as `null`, which only widens one bucket; -/// [`same_serialized_expression`] decides. -pub(super) fn expression_hash(expression: &Operator, serialized: Option<&str>) -> u64 { +/// A hash consistent with [`same_serialized_expression`], used only to +/// narrow the groups compared: SipHash over the expression's +/// [`serialized_expression`]. A non-finite `f64` serializes as `null`, +/// which only widens one bucket; [`same_serialized_expression`] decides. +pub(super) fn expression_hash(serialized: &str) -> u64 { let mut hasher = DefaultHasher::new(); - serialized.unwrap_or_default().hash(&mut hasher); - expression.children().hash(&mut hasher); + serialized.hash(&mut hasher); hasher.finish() } -/// Whether `left` and `right` are one expression: equal operators and -/// arguments, by typed equality confirmed by equal serializations (each -/// side's [`serialized_operator`], computed once by the caller), over the -/// same input groups at the same positions. +/// Whether `left` and `right` are one expression: typed equality (operator, +/// arguments and input groups) confirmed by equal serializations (each +/// side's [`serialized_expression`], computed once by the caller), as +/// [`same_value`] decides it. pub(super) fn same_serialized_expression( - (left, left_serialized): (&Operator, Option<&str>), - (right, right_serialized): (&Operator, Option<&str>), + (left, left_serialized): (&Operator, &str), + (right, right_serialized): (&Operator, &str), ) -> bool { - left.children() == right.children() - && operator_and_arguments(left) == operator_and_arguments(right) - && left_serialized.is_some() - && left_serialized == right_serialized -} - -/// [`same_serialized_expression`], serializing both sides. -fn same_expression(left: &Operator, right: &Operator) -> bool { - same_serialized_expression( - (left, serialized_operator(left).as_deref()), - (right, serialized_operator(right).as_deref()), - ) + left == right && left_serialized == right_serialized } /// Typed equality, confirmed by equal serializations: numeric `PartialEq` @@ -78,13 +60,13 @@ pub(super) fn same_value(left: &T, right: &T) - } /// Whether `left` and `right` are one candidate of a group: the same -/// expression ([`same_expression`]) and logical properties (parameters by -/// [`same_value`]). +/// expression and logical properties (parameters), each by [`same_value`]: +/// typed equality first, serialized only when typed-equal. pub(super) fn same_expression_and_logical_properties( left: &CandidateForMemoGroup, right: &CandidateForMemoGroup, ) -> bool { - same_expression(&left.expression.operator, &right.expression.operator) + same_value(&left.expression.operator, &right.expression.operator) && same_value( &left.logical_properties.parameters, &right.logical_properties.parameters, diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 101e89f3d..331f69afc 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -20,7 +20,7 @@ use crate::stages::{ use crate::workload::{DataWorkload, SourceFacts}; use hash_consing::{ expression_hash, same_expression_and_logical_properties, same_serialized_expression, - serialized_operator, + serialized_expression, }; /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with @@ -31,12 +31,13 @@ pub struct Memo { /// The groups, by id ([`Self::groups`]); added only through /// [`Self::find_or_add_group`]. groups: Vec, - /// The queries, by index in the workload. - pub queries: Vec, + /// The queries, by index in the workload ([`Self::queries`]); added only + /// through [`Self::add_query`]. + queries: Vec, /// The groups [`Self::find_or_add_group`] may return, by /// [`expression_hash`] of their expression: every group whose sharing /// is legal ([`GroupProperties::shareable_when_equal`](crate::stages::GroupProperties::shareable_when_equal)). - groups_by_expression_hash: HashMap)>>, + groups_by_expression_hash: HashMap>, /// The data workload's facts per source (`DataWorkload::sources`), fixed /// at construction: a `Scan` group's column signs read them. pub(super) source_facts: BTreeMap, @@ -74,6 +75,11 @@ impl Memo { &self.groups } + /// The queries, by index in the workload. + pub fn queries(&self) -> &[MemoQuery] { + &self.queries + } + /// Adds `query` and returns its index: its root's groups and every /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 /// adds each query once. @@ -85,8 +91,9 @@ impl Memo { index } - /// Hash-consing: the group whose expression is `expression` - /// (`same_expression`: operator, arguments and input groups), else a + /// Hash-consing: the group whose expression is `expression` (operator, + /// arguments and input groups by typed equality, confirmed by equal + /// serializations: `same_serialized_expression`), else a /// new group for it, with the `GroupProperties` `derive_group_properties` /// derives from its input groups, listed as a parent of each input /// group. A group whose copies may return different rows (an unordered @@ -99,8 +106,8 @@ impl Memo { &mut self, expression: Operator, ) -> Result { - let serialized = serialized_operator(&expression); - let hash = expression_hash(&expression, serialized.as_deref()); + let serialized = serialized_expression(&expression); + let hash = expression_hash(&serialized); // Only shareable groups are listed by hash, and an equal expression // is as shareable as the group it equals, which was validated. if let Some((group, _)) = self @@ -109,8 +116,8 @@ impl Memo { .and_then(|groups| { groups.iter().find(|(group, known)| { same_serialized_expression( - (&self.groups[group.0 as usize].expression, known.as_deref()), - (&expression, serialized.as_deref()), + (&self.groups[group.0 as usize].expression, known), + (&expression, &serialized), ) }) }) @@ -175,7 +182,7 @@ impl Memo { memo: self, group_id, }); - let schema = validate_expression_over_input_groups(&operator) + let (schema, _) = validate_expression_over_input_groups(&operator) .map_err(CandidateError::NotValidOverInputGroups)?; let result_kind = operator.output_kind(); let group_properties = &self.groups[group.0 as usize].group_properties; diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 8376f86d6..99a3d7ba9 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -5,7 +5,6 @@ use super::*; use crate::ir::operator::arguments::{GroupKeys, Reduction, Source}; use crate::ir::operator::AggIntent; -use crate::ir::scalar::expr_ir::ColumnRef; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{ProjectItem, SortKey}; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; @@ -13,36 +12,8 @@ use crate::ir::ASAPOp; use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; use crate::stages::logical::MemoGroupAsInput; use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; -use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate}; - -/// `Scan metric[1m]`: `[ts, value]`. -fn scan(metric: &str) -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - }) -} - -/// `SummaryAgg[KLL](input)`: no θ in the expression. -fn kll_state(input: MemoGroupId) -> Operator { - Operator::ASAP(ASAPOp::SummaryAgg { - child: input, - summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - filter: None, - }) -} +use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use crate::test_support::memo_fixtures::{scan, state}; /// `SummaryEstimate[q](state)`. fn quantile_readout(state: MemoGroupId, q: f64) -> Operator { @@ -110,11 +81,26 @@ fn kll_k(k: u32) -> Option { #[test] fn find_or_add_group_returns_the_group_of_an_equal_expression() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); - assert_eq!(memo.find_or_add_group(scan("latency")).unwrap(), g1); - assert_eq!(memo.find_or_add_group(kll_state(g1)).unwrap(), g2); - assert_ne!(memo.find_or_add_group(scan("errors")).unwrap(), g1); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + assert_eq!( + memo.find_or_add_group(scan("latency", &[], vec![])) + .unwrap(), + g1 + ); + assert_eq!( + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(), + g2 + ); + assert_ne!( + memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(), + g1 + ); assert_eq!(memo.groups().len(), 3); assert_eq!( memo.groups()[g2.0 as usize].expression.children(), @@ -126,11 +112,15 @@ fn find_or_add_group_returns_the_group_of_an_equal_expression() { #[test] fn find_or_add_group_separates_equal_operators_over_different_inputs() { let mut memo = Memo::new(&DataWorkload::default()); - let latency = memo.find_or_add_group(scan("latency")).unwrap(); - let errors = memo.find_or_add_group(scan("errors")).unwrap(); + let latency = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let errors = memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(); assert_ne!( - memo.find_or_add_group(kll_state(latency)).unwrap(), - memo.find_or_add_group(kll_state(errors)).unwrap() + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), latency)) + .unwrap(), + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), errors)) + .unwrap() ); } @@ -139,7 +129,9 @@ fn find_or_add_group_separates_equal_operators_over_different_inputs() { #[test] fn find_or_add_group_never_returns_an_unshareable_group() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); assert_ne!( memo.find_or_add_group(unordered_limit(g1)).unwrap(), memo.find_or_add_group(unordered_limit(g1)).unwrap() @@ -150,9 +142,14 @@ fn find_or_add_group_never_returns_an_unshareable_group() { #[test] fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); - memo.find_or_add_group(kll_state(g1)).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); assert_eq!(memo.groups()[g1.0 as usize].parent_groups(), &[g2]); assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); } @@ -162,14 +159,32 @@ fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { #[test] fn kll_builds_with_k_200_and_k_345_share_one_group() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let k200_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let k200_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); let k200 = memo - .add_candidate(k200_group, candidate(kll_state(g1), kll_k(200))) + .add_candidate( + k200_group, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(200), + ), + ) + .unwrap(); + let k345_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) .unwrap(); - let k345_group = memo.find_or_add_group(kll_state(g1)).unwrap(); let k345 = memo - .add_candidate(k345_group, candidate(kll_state(g1), kll_k(345))) + .add_candidate( + k345_group, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(345), + ), + ) .unwrap(); assert_eq!(k200_group, k345_group); assert_ne!(k200, k345); @@ -185,13 +200,29 @@ fn kll_builds_with_k_200_and_k_345_share_one_group() { #[test] fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); let first = memo - .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .add_candidate( + g2, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(200), + ), + ) .unwrap(); let again = memo - .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .add_candidate( + g2, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(200), + ), + ) .unwrap(); assert_eq!(first, again); assert_eq!( @@ -201,7 +232,10 @@ fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { index_in_group: 0 } ); - let other_size = candidate(kll_state(g1), kll_k(345)); + let other_size = candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(345), + ); assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); } @@ -212,14 +246,22 @@ fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { fn add_candidate_maintains_parents_and_queries() { // Query 0 is G3 = SummaryEstimate[q=0.95](G2) over G2 = KLL(G1). let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); assert_eq!(memo.add_query(query(g3)), 0); // A rule adds a state group G5 over another scan G4: no query // reaches them yet. - let g4 = memo.find_or_add_group(scan("latency_copy")).unwrap(); - let g5 = memo.find_or_add_group(kll_state(g4)).unwrap(); + let g4 = memo + .find_or_add_group(scan("latency_copy", &[], vec![])) + .unwrap(); + let g5 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g4)) + .unwrap(); assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); // G3 gains a candidate reading G5: G5 and G4 now serve query 0. memo.add_candidate(g3, candidate(quantile_readout(g5, 0.95), None)) @@ -247,8 +289,12 @@ fn add_candidate_maintains_parents_and_queries() { #[test] fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); let before = format!("{memo:?}"); assert!(matches!( memo.add_candidate(g2, candidate(quantile_readout(g2, 0.5), None)), @@ -266,8 +312,12 @@ fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { #[test] fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); let over_raw_rows = quantile_readout(g1, 0.95).map_children(|&group_id| MemoGroupAsInput { memo: &memo, @@ -294,7 +344,9 @@ fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { #[test] fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); let out_of_range = Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::by(vec![9]), measures: vec![crate::ir::operator::agg_intent::AggIntent::Sum { col: Some(1) }], diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 9322812c3..2ca021553 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -10,13 +10,15 @@ mod memo; mod memo_candidates; mod requirements; +pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; pub use candidate::{ CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, }; +pub use column_properties::ValueSign; pub use edge::LogicalEdge; use group::MemoGroupAsInput; -pub use group::{GroupProperties, MemoGroup, MemoGroupId, ValueSign}; +pub use group::{GroupProperties, MemoGroup, MemoGroupId}; pub use memo::{queries_rooted_at, root_groups, Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; pub use requirements::{LogicalRequirements, RequiredAccuracy}; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 4ad565d9f..180f2ad09 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -13,10 +13,11 @@ mod physical; mod pruning; pub use logical::{ - queries_rooted_at, root_groups, CandidateError, CandidateExpression, CandidateForMemoGroup, - CandidateForMemoGroupId, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, - LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupId, - MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, + queries_rooted_at, root_groups, AggregateCoverage, AggregateDefinition, CandidateError, + CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, CoverageError, + GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, + LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupId, MemoQuery, + OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, }; pub use per_computation_logical_memo_candidates::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index cd646f4e8..5f0605a84 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -4,10 +4,12 @@ //! to the crate-private function of the same name, except the fixture //! [`population_over_selector`], which the executor's tests build. +pub mod memo_fixtures; + use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; -use crate::ir::properties::aggregate_coverage::AggregateCoverage; use crate::plan::materialized_view::RawDataWindow; +use crate::stages::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::{ASAPOp, NonASAPOp}; diff --git a/crates/types/tests/memo_fixtures.rs b/crates/types/src/test_support/memo_fixtures.rs similarity index 78% rename from crates/types/tests/memo_fixtures.rs rename to crates/types/src/test_support/memo_fixtures.rs index b64386ceb..351a8dbf1 100644 --- a/crates/types/tests/memo_fixtures.rs +++ b/crates/types/src/test_support/memo_fixtures.rs @@ -1,13 +1,14 @@ -//! The memo expressions `memo_group_properties` and `memo_aggregate_coverage` -//! build their examples from. +//! Memo expressions over group ids that the memo's unit tests and this +//! crate's memo integration tests (`memo_group_properties`, +//! `memo_aggregate_coverage`) build their examples from. use std::time::Duration; -use asap_types::ir::operator::arguments::{AtModifier, Reduction, Source, TimeShift}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; -use asap_types::stages::MemoGroupId; -use asap_types::summary_type::SummaryUpdate; +use crate::ir::operator::arguments::{AtModifier, Reduction, Source, TimeShift}; +use crate::ir::scalar::ColumnRef; +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::ir::{ASAPOp, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; +use crate::stages::MemoGroupId; +use crate::summary_type::SummaryUpdate; /// `Scan metric{predicates}`: `[ts, value, columns…]`, `ts` its time index. pub fn scan( diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index 7bce47c42..ad9e48481 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -3,7 +3,6 @@ //! One file: `cargo test -p asap-types --test types ::`. mod memo_aggregate_coverage; -mod memo_fixtures; mod memo_group_properties; mod physical_export; mod planner_vocabulary; diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index 830184c70..457dc0dbf 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -1,4 +1,4 @@ -//! Group properties and aggregate coverage, derived once per memo group +//! Aggregate coverage (`AggregateCoverage`), derived once per memo group //! from its input groups (`docs/design_docs/proposals/asap-primitive-schema.md` //! §4.2.2). The memo of the worked example: //! @@ -13,22 +13,21 @@ use std::ops::Bound; use asap_types::ir::operator::arguments::{AtModifier, Reduction}; use asap_types::ir::operator::AggIntent; -use crate::memo_fixtures::{filter, range, scan, shift, state}; -use asap_types::ir::properties::aggregate_coverage::{ - AggregateCoverage, ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, -}; +use asap_types::ir::properties::selection::{ColumnIdentity, Constraint, IntervalConstraintLimit}; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, SchemaDerivationError, }; +use asap_types::stages::{AggregateCoverage, CoverageError}; use asap_types::stages::{ CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, }; use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::definition_id; +use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; use asap_types::workload::DataWorkload; const TS: usize = 0; diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index fbec6f122..5d647eee6 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -11,10 +11,8 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::ir::operator::arguments::{AtModifier, GroupKeys, JoinKind, Reduction, Source}; -use asap_types::ir::operator::input::scans_only_time_series; use asap_types::ir::operator::AggIntent; -use crate::memo_fixtures::{filter, range, scan, shift, state}; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ProjectItem; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; @@ -23,6 +21,7 @@ use asap_types::ir::{ }; use asap_types::stages::{Memo, MemoGroupId, ValueSign}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; +use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; use ValueSign::{NonNegative, NonPositive, Unknown}; @@ -430,7 +429,7 @@ fn a_result_depends_on_the_evaluation_time_under_a_time_range_or_now() { /// `scanned_sources`, derived once per group: a `Scan` its source, any other /// group the union over its input groups. A PromQL `Scan` of a time series /// and a `TimeRange` over it scan only time series -/// (`scans_only_time_series`); a SQL table `Scan` does not, nor a `Join` +/// (`Source::is_time_series`); a SQL table `Scan` does not, nor a `Join` /// reading two (a SQL join reads relations only, so a time series cannot be /// one of its inputs). #[test] @@ -475,7 +474,12 @@ fn a_group_scans_the_sources_of_the_scans_below_it() { metric: "latency".into() }]) ); - let reads = |group: MemoGroupId| scans_only_time_series(&properties(group).scanned_sources); + let reads = |group: MemoGroupId| { + properties(group) + .scanned_sources + .iter() + .all(Source::is_time_series) + }; assert!(reads(series)); assert!(reads(window)); assert!(!reads(requests)); @@ -510,28 +514,19 @@ fn count_values_without_has_one_sign_per_output_column() { /// readout's value column, and whether equal copies may be one group. #[test] fn group_properties_come_from_input_groups() { - let mut memo = Memo::new(&DataWorkload::default()); - let g1 = memo - .find_or_add_group(scan( - "latency", - &[ - ("job", DataType::Utf8), - ("a", DataType::Float64), - ("b", DataType::Float64), - ], - vec![], - )) - .unwrap(); + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); let api = compare( - ScalarExpr::Column(2), + ScalarExpr::Column(HOST), CompareOpKind::Eq, ScalarExpr::Literal(ScalarValue::Utf8("api".into())), ); - let g2 = memo.find_or_add_group(filter(api, g1)).unwrap(); - let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); - let g4 = memo - .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g3)) - .unwrap(); + let g2 = add(&mut memo, filter(api, g1)); + let g3 = add(&mut memo, range(MINUTE_MS, g2)); + let g4 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g3), + ); let readout = memo .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: g4, @@ -539,7 +534,7 @@ fn group_properties_come_from_input_groups() { })) .unwrap(); let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; - assert_eq!(properties(g1).schema.fields.len(), 5); + assert_eq!(properties(g1).schema.fields.len(), 3); assert_eq!(properties(g1).aggregate_coverage, None); assert_eq!(properties(g4).result_kind, OperatorResultKind::State); assert!(properties(g4) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index d6d5479bb..179eb6f6e 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -192,7 +192,7 @@ GROUP BY job; **Existing implementation.** The `WHERE` split is implemented for DataFusion in [`datafusion-contrib/datafusion-materialized-views`](https://github.com/datafusion-contrib/datafusion-materialized-views), `src/rewrite/normal_form.rs` (`SpjNormalForm`, `Predicate { eq_classes, ranges_by_equivalence_class, residuals }`). It rejects plans that contain an `Aggregate` or a `Join`. -### 4.2 Summary Coverage = Summary definition + selection +### 4.2 Aggregate coverage = aggregate definition + selection #### 4.2.1 Why coverage has two parts @@ -1373,7 +1373,7 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Summary coverage (`crates/types/src/ir/properties/aggregate_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. From c2f23d760bb8fffaee38eb66bfb10f0dba02a79d Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:34:25 +0000 Subject: [PATCH 08/12] Sign rules: no sign for sign-flipping sketch estimates and out-of-range quantiles; arithmetic signs - ValueSign documents its meaning: NonNegative means every non-NaN value is >= 0, NonPositive every non-NaN value is <= 0. - A point count or top-k read from a Count Sketch or UnivMon state (and their variants) is Unknown: a median of signed counters can be negative over non-negative weights. SummaryTypeCapability::estimates_may_flip_sign holds the per-algorithm fact; Count-Min keeps the state's sign. - A quantile with q outside [0, 1] is Unknown, for both AggIntent::Quantile and SketchStatistic::Quantile. - Vector BinaryOp arithmetic and ScalarExpr arithmetic share one rule (arithmetic_sign): a sum of two non-negatives (non-positives) is non-negative (non-positive); a product or quotient of same signs is non-negative and of opposite signs non-positive; anything else Unknown. - PromqlVectorFromScalar(expr)'s value has expression_sign(expr). --- .../src/stages/logical/column_properties.rs | 96 +++++-- crates/types/src/summary_type/capability.rs | 24 ++ crates/types/tests/memo_group_properties.rs | 241 +++++++++++++++++- 3 files changed, 335 insertions(+), 26 deletions(-) diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index ee5deaeff..0a24b7c06 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -10,12 +10,14 @@ use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKi use crate::ir::operator::asap::{summary_agg_layout, summary_build}; use crate::ir::operator::input::OperatorInput; use crate::ir::operator::AggIntent; -use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::scalar::expr_ir::{ArithmeticOpKind, ScalarValue}; use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; use crate::ir::schema::{FieldDataType, Schema}; use crate::ir::{ASAPOp, NonASAPOp, Operator}; -use crate::summary_type::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; +use crate::summary_type::{ + ExactKind, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, +}; use crate::workload::bounds::source_name; use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; @@ -26,8 +28,11 @@ use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] pub enum ValueSign { + /// Every non-NaN value is ≥ 0. NonNegative, + /// Every non-NaN value is ≤ 0. NonPositive, + /// Neither is known. Unknown, } @@ -88,7 +93,15 @@ fn non_asap_signs( }) .collect() } - Values { .. } | PromqlVectorFromScalar(_) => unknown(), + Values { .. } => unknown(), + // The scalar's value, every other column `Unknown`. + PromqlVectorFromScalar(expr) => { + let mut signs = unknown(); + if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, output) { + signs[value] = expression_sign(expr, &[]); + } + signs + } Filter { child, .. } | Sort { child, .. } | Limit { child, .. } @@ -157,7 +170,7 @@ fn non_asap_signs( ) => signs[value], BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(signs[value], rhs_value), BinaryOpKind::CompareBool(_) => NonNegative, - BinaryOpKind::Arithmetic(_) => Unknown, + BinaryOpKind::Arithmetic(op) => arithmetic_sign(op, signs[value], rhs_value), }; } signs.resize(output.fields.len(), Unknown); @@ -230,14 +243,26 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { SummaryEstimate { summary_input, query, - } => readout_signs(summary_input, output, |state| match query { - SketchStatistic::Cardinality - | SketchStatistic::FrequencyL2 - | SketchStatistic::FrequencyEntropy => NonNegative, - SketchStatistic::Quantile { .. } - | SketchStatistic::PointCount { .. } - | SketchStatistic::TopK { .. } => state, - }), + } => { + // The summary type of the state read, for whether its counts keep + // the weights' sign. + let flips = summary_input + .schema() + .fields + .iter() + .find(|field| !field.is_plain()) + .is_some_and(|state| SummaryTypeCapability::estimates_may_flip_sign(&state.dtype)); + readout_signs(summary_input, output, |state| match query { + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy => NonNegative, + SketchStatistic::Quantile { q } => quantile_sign(*q, state), + SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } if flips => { + Unknown + } + SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } => state, + }) + } FinalizeExactAccumulator { child } => { let counts = child.schema().fields.iter().any(|field| { matches!( @@ -270,8 +295,8 @@ fn project_signs(cols: &[ProjectItem], input: &[ValueSign]) -> Vec(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { match expr { ScalarExpr::Column(column) => input.get(*column).copied().unwrap_or(Unknown), @@ -279,6 +304,13 @@ fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { ScalarExpr::Literal(ScalarValue::Int64(value)) => sign_of_number(*value as f64), ScalarExpr::Negative { expr, .. } => flipped(expression_sign(expr, input)), ScalarExpr::Cast { expr, .. } => expression_sign(expr, input), + ScalarExpr::Arithmetic { + op, left, right, .. + } => arithmetic_sign( + op, + expression_sign(left, input), + expression_sign(right, input), + ), // PromQL's `abs(v)` lowers to `promql_abs`, SQL's to `abs`. ScalarExpr::FunctionCall { name, .. } if name.eq_ignore_ascii_case("abs") || name == "promql_abs" => @@ -352,13 +384,8 @@ fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { | FrequencyL2 { .. } | FrequencyEntropy { .. } | Math(crate::ir::operator::MathFunc::Abs) => NonNegative, - Sum { .. } - | Avg { .. } - | Min { .. } - | Max { .. } - | Quantile { .. } - | LastOverTime - | FirstOverTime => input, + Sum { .. } | Avg { .. } | Min { .. } | Max { .. } | LastOverTime | FirstOverTime => input, + Quantile { q, .. } => quantile_sign(*q, input), // A counter's rate and increase are non-negative; over a value of // another sign a decrease reads as a reset, so no sign is known. Rate | IRate | Increase if input == NonNegative => NonNegative, @@ -409,6 +436,33 @@ fn readout_signs( .collect() } +/// The sign of `left op right`: a sum of two values of one sign has that +/// sign; a product or quotient of two values of one sign is non-negative, +/// of opposite signs non-positive; any other operator, or an operand of +/// unknown sign, gives `Unknown`. PromQL's vector arithmetic and scalar +/// expressions both read it. +fn arithmetic_sign(op: &ArithmeticOpKind, left: ValueSign, right: ValueSign) -> ValueSign { + match (op, left, right) { + (ArithmeticOpKind::Add, NonNegative, NonNegative) => NonNegative, + (ArithmeticOpKind::Add, NonPositive, NonPositive) => NonPositive, + (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonNegative, NonNegative) + | (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonPositive, NonPositive) => NonNegative, + (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonNegative, NonPositive) + | (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonPositive, NonNegative) => NonPositive, + _ => Unknown, + } +} + +/// A quantile's sign: the input's for 0 ≤ q ≤ 1; PromQL gives ±Inf +/// outside it, so `Unknown`. +fn quantile_sign(q: f64, input: ValueSign) -> ValueSign { + if (0.0..=1.0).contains(&q) { + input + } else { + Unknown + } +} + /// Per column, the inputs' sign if they all agree, else `Unknown`. fn agreed(inputs: &[Vec]) -> Vec { let Some((first, rest)) = inputs.split_first() else { diff --git a/crates/types/src/summary_type/capability.rs b/crates/types/src/summary_type/capability.rs index 612f854ae..63f873244 100644 --- a/crates/types/src/summary_type/capability.rs +++ b/crates/types/src/summary_type/capability.rs @@ -110,6 +110,30 @@ impl SummaryTypeCapability { ) && !Self::shares_one_state_across_groups(summary_type) } + /// Whether a point count or top-k read from a state of `summary_type` + /// may have a sign the weights it was updated with do not: a median of + /// signed counters (Count Sketch, UnivMon and their variants) can be + /// negative over non-negative weights; an overestimating counter + /// (Count-Min) or an exact count cannot. + pub fn estimates_may_flip_sign(summary_type: &FieldDataType) -> bool { + use SketchAlgorithm as A; + matches!( + summary_type, + FieldDataType::Sketch( + A::CountSketch + | A::CountSketchFold + | A::CountSketchWithHeap + | A::CountSketchWithL2Norm + | A::CountSketchWithSpaceSaving + | A::HydraCountSketch + | A::UnivMon + | A::UnivMonQ + | A::UnivMonTwoTier + | A::HydraUnivMon + ) + ) + } + /// Whether `summary_type` is a Hydra algorithm: one structure shared by /// every group, not one state per group. pub fn shares_one_state_across_groups(summary_type: &FieldDataType) -> bool { diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 5d647eee6..db6e69082 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -10,14 +10,17 @@ //! ``` use std::collections::{BTreeMap, BTreeSet}; -use asap_types::ir::operator::arguments::{AtModifier, GroupKeys, JoinKind, Reduction, Source}; +use asap_types::ir::operator::arguments::{ + AtModifier, BinaryOpKind, GroupKeys, JoinKind, Reduction, Source, +}; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use asap_types::ir::scalar::ProjectItem; +use asap_types::ir::scalar::{ColumnRef, ProjectItem}; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, + ScalarExpr, }; use asap_types::stages::{Memo, MemoGroupId, ValueSign}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; @@ -144,7 +147,8 @@ fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator /// | G6 | `Project(abs(value))(G2)` | value `NonNegative` | /// | G7 | `Project(-value)(G1)` | value `NonPositive` (flipped) | /// | G8 | `Project(-value)(G7)` | value `NonNegative` (flipped back) | -/// | G9 | `Project(value * 2)(G1)` | value `Unknown` (another expression) | +/// | G9 | `Project(value * 2)(G1)` | value `NonNegative` (same signs multiply) | +/// | G10 | `Project(value - value)(G1)` | value `Unknown` (a difference) | #[test] fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { let mut memo = memo(); @@ -192,6 +196,18 @@ fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { g1, ), ); + let g10 = add( + &mut memo, + map_value( + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Sub, + left: value(), + right: value(), + semantics: ExprSemantics::Promql, + }, + g1, + ), + ); let declared = [Unknown, NonNegative, Unknown]; assert_eq!(signs(&memo, g1), declared); assert_eq!(signs(&memo, g2), [Unknown; 3]); @@ -202,7 +218,8 @@ fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { (g6, NonNegative), (g7, NonPositive), (g8, NonNegative), - (g9, Unknown), + (g9, NonNegative), + (g10, Unknown), ] { assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{group:?}"); } @@ -555,3 +572,217 @@ fn group_properties_come_from_input_groups() { && field.dtype == FieldDataType::Plain(DataType::Float64))); assert!(properties(g4).shareable_when_equal); } + +/// A point count read from a Count Sketch is a median of signed counters, +/// which can be negative over non-negative weights: `Unknown`; one read +/// from a Count-Min state, an overestimate of a sum of non-negative +/// weights, keeps the weights' sign. +#[test] +fn a_count_sketch_point_count_has_no_sign_a_count_min_one_keeps_it() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, range(MINUTE_MS, g1)); + let point_count = |memo: &mut Memo, algorithm| { + let state = add(memo, state(FieldDataType::Sketch(algorithm), g2)); + add( + memo, + readout( + SketchStatistic::PointCount { + key: ColumnRef::Named("host".into()), + value: Some("a".into()), + }, + state, + ), + ) + }; + let count_sketch = point_count(&mut memo, SketchAlgorithm::CountSketch); + let count_min = point_count(&mut memo, SketchAlgorithm::Cms); + // [ts, count, host]: the count is the state's column. + let value = |group: MemoGroupId| { + let schema = &memo.groups()[group.0 as usize].group_properties.schema; + signs(&memo, group)[schema.column_id("count").unwrap()] + }; + assert_eq!(value(count_sketch), Unknown); + assert_eq!(value(count_min), NonNegative); +} + +/// PromQL's `quantile(φ, v)` with φ outside [0, 1] is ±Inf, so a quantile +/// keeps its input's sign only for 0 ≤ φ ≤ 1, as a measure and as a +/// sketch readout. +#[test] +fn a_quantile_outside_the_unit_interval_has_no_sign() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let quantile = |memo: &mut Memo, q| { + add( + memo, + aggregate( + by_host(), + AggIntent::Quantile { + col: Some(VALUE), + q, + }, + g1, + ), + ) + }; + let inside = quantile(&mut memo, 0.5); + let below = quantile(&mut memo, -0.5); + let g2 = add(&mut memo, range(MINUTE_MS, g1)); + let kll = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g2), + ); + let read = add( + &mut memo, + readout(SketchStatistic::Quantile { q: 1.5 }, kll), + ); + let value = |group: MemoGroupId| *signs(&memo, group).last().unwrap(); + assert_eq!(value(inside), NonNegative); + assert_eq!(value(below), Unknown); + assert_eq!(value(read), Unknown); +} + +fn arithmetic(op: ArithmeticOpKind, lhs: MemoGroupId, rhs: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(op), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) +} + +/// Arithmetic between two vectors: a sum of non-negatives, and a product or +/// quotient of same-signed values, is non-negative; a difference has no +/// sign. A `vector(s)` value has `s`'s sign: +/// +/// | Group | Expression | value sign | +/// |---|---|---| +/// | G3 | `quantile_over_time(0.9, G2)`, G2 = `latency[1h]` | `NonNegative` | +/// | G4 | `quantile_over_time(0.5, G2)` | `NonNegative` | +/// | G5 | `G3 / G4` | `NonNegative` (was `Unknown`) | +/// | G6 | `G3 + G4` | `NonNegative` | +/// | G7 | `G3 - G4` | `Unknown` | +/// | G8 | `vector(1000)` | `NonNegative` (was `Unknown`) | +/// | G9 | `vector(-1)` | `NonPositive` | +/// | G10 | `G3 * G8` | `NonNegative` | +/// | G11 | `G3 * G9` | `NonPositive` | +/// | G12 | `G9 * G9` | `NonNegative` | +#[test] +fn vector_arithmetic_follows_its_operands_signs() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, range(60 * MINUTE_MS, g1)); + let quantile = |q| AggIntent::Quantile { + col: Some(VALUE), + q, + }; + let g3 = add( + &mut memo, + aggregate(Reduction::PerEntity, quantile(0.9), g2), + ); + let g4 = add( + &mut memo, + aggregate(Reduction::PerEntity, quantile(0.5), g2), + ); + let g5 = add(&mut memo, arithmetic(ArithmeticOpKind::Div, g3, g4)); + let g6 = add(&mut memo, arithmetic(ArithmeticOpKind::Add, g3, g4)); + let g7 = add(&mut memo, arithmetic(ArithmeticOpKind::Sub, g3, g4)); + let vector = |s: f64| { + Operator::NonASAP(NonASAPOp::PromqlVectorFromScalar(ScalarExpr::Literal( + ScalarValue::Float64(s), + ))) + }; + let g8 = add(&mut memo, vector(1000.0)); + let g9 = add(&mut memo, vector(-1.0)); + let g10 = add(&mut memo, arithmetic(ArithmeticOpKind::Mul, g3, g8)); + let g11 = add(&mut memo, arithmetic(ArithmeticOpKind::Mul, g3, g9)); + let g12 = add(&mut memo, arithmetic(ArithmeticOpKind::Mul, g9, g9)); + let value = |memo: &Memo, group: MemoGroupId| { + let schema = &memo.groups()[group.0 as usize].group_properties.schema; + signs(memo, group)[schema.column_id("value").unwrap()] + }; + for (group, sign) in [ + (g3, NonNegative), + (g4, NonNegative), + (g5, NonNegative), + (g6, NonNegative), + (g7, Unknown), + (g8, NonNegative), + (g9, NonPositive), + (g10, NonNegative), + (g11, NonPositive), + (g12, NonNegative), + ] { + assert_eq!(value(&memo, group), sign, "{group:?}"); + } +} + +/// A vector-scalar operation lowers to a `Project` of an arithmetic +/// expression (PromQL `x / 1000`, `x * 8`), as does SQL's `value * 8`; the +/// expression's sign follows the same rule as vector arithmetic: +/// +/// | Group | Expression | value sign | +/// |---|---|---| +/// | G2 | `G1 / 1000`, G1 = `Scan latency` (PromQL) | `NonNegative` (was `Unknown`) | +/// | G3 | `G1 * 8` (PromQL) | `NonNegative` (was `Unknown`) | +/// | G4 | `SELECT value * 8 FROM G1` (SQL) | `NonNegative` (was `Unknown`) | +/// | G5 | `G1 - G1` | `Unknown` | +/// | G6 | `G1 * -8` | `NonPositive` | +#[test] +fn scalar_arithmetic_follows_its_operands_signs() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let expr = |op, right: ScalarExpr, semantics| ScalarExpr::Arithmetic { + op, + left: Box::new(ScalarExpr::Column(VALUE)), + right: Box::new(right), + semantics, + }; + let number = |n: f64| ScalarExpr::Literal(ScalarValue::Float64(n)); + let promql = ExprSemantics::Promql; + let g2 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Div, number(1000.0), promql), g1), + ); + let g3 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Mul, number(8.0), promql), g1), + ); + let g4 = add( + &mut memo, + map_value( + expr( + ArithmeticOpKind::Mul, + ScalarExpr::Literal(ScalarValue::Int64(8)), + ExprSemantics::Sql, + ), + g1, + ), + ); + let g5 = add( + &mut memo, + map_value( + expr(ArithmeticOpKind::Sub, ScalarExpr::Column(VALUE), promql), + g1, + ), + ); + let g6 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Mul, number(-8.0), promql), g1), + ); + for (group, value) in [ + (g2, NonNegative), + (g3, NonNegative), + (g4, NonNegative), + (g5, Unknown), + (g6, NonPositive), + ] { + assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{group:?}"); + } +} From 73785fa075d515cb45b0df4accbe3e52cbe9deae Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:41:59 +0000 Subject: [PATCH 09/12] Two scopes, two identities for an aggregate definition - The memo compares coverages by AggregateDefinition == (it holds memo group ids); AggregateCoverage::definition_id and its test_support helper are deleted, and the memo coverage test compares definitions only. - ViewCoverage::definition_id is documented as the plan-scope content id: the definition with every input group expanded recursively to its expression down to the Scans, with no group or node ids, canonically serialized and hashed. coverage_of (still todo for PR 6) returns the ViewCoverage and names that content id. - Design docs (asap-primitive-schema.md, physical-planning-and-deployment.md) say the memo compares definitions and the plan carries the content id. - Constraint::Interval documents that a full-range interval is still a constraint: it excludes NULL and NaN rows. --- crates/types/src/ir/properties/selection.rs | 5 +++ crates/types/src/plan/materialized_view.rs | 36 ++++++++++--------- .../src/stages/logical/aggregate_coverage.rs | 19 +--------- crates/types/src/test_support.rs | 6 ---- crates/types/tests/memo_aggregate_coverage.rs | 7 +--- .../physical-planning-and-deployment.md | 15 +++++--- .../proposals/asap-primitive-schema.md | 7 ++-- 7 files changed, 40 insertions(+), 55 deletions(-) diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs index 5456a0069..7b9351594 100644 --- a/crates/types/src/ir/properties/selection.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -134,6 +134,11 @@ pub enum Constraint { /// | PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | /// | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | /// | SQL `WHERE price > 10` (column `price`) | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | + /// + /// A full-range interval `(Unbounded, Unbounded)` is still a + /// constraint, not the absence of one: it keeps every ordered value and + /// excludes the rows whose value is NULL or NaN. A column with no + /// constraint in a box keeps every row. Interval { #[serde(with = "limit_bound")] lower: Bound, diff --git a/crates/types/src/plan/materialized_view.rs b/crates/types/src/plan/materialized_view.rs index 27ff1a3af..33fb6f22e 100644 --- a/crates/types/src/plan/materialized_view.rs +++ b/crates/types/src/plan/materialized_view.rs @@ -78,8 +78,8 @@ use crate::ir::operator::node::{Operator, OperatorNode}; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::selection::SelectionBox; +use crate::stages::CoverageError; use crate::stages::Lifetime; -use crate::stages::{AggregateCoverage, CoverageError}; /// One materialized view of a DAG: what its stored state is, what it is /// refreshed from, when, and for how long it is retained. @@ -121,15 +121,20 @@ impl MaterializedView { } /// A view's aggregate coverage -/// ([`AggregateCoverage`], `docs/design_docs/proposals/asap-primitive-schema.md` +/// ([`AggregateCoverage`](crate::stages::AggregateCoverage), `docs/design_docs/proposals/asap-primitive-schema.md` /// §4.2): what it computes and which rows went in. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ViewCoverage { - /// `AggregateCoverage::definition_id`: equal on every node, plan and - /// process that computes the same thing, whatever its selection; a - /// rollup to a coarser grouping computes something else and has its - /// own. + /// The plan-scope content id of what the view computes: its memo + /// coverage's `AggregateDefinition` with every input group expanded + /// recursively to its expression, down to the `Scan`s, holding no + /// memo group or plan node ids, canonically serialized and hashed. + /// Equal across plans and processes that compute the same thing, + /// whatever the selection, and different when they do not (a rollup to + /// a coarser grouping computes something else and has its own). Inside + /// one memo, coverages compare by `AggregateDefinition` `==` instead + /// and need no id. pub(crate) definition_id: String, /// The rows that went in: a union of boxes of value sets, value ranges /// and time relative to the evaluation time. A merge's is the union of @@ -139,7 +144,7 @@ pub struct ViewCoverage { #[cfg(any(test, feature = "test-support"))] impl ViewCoverage { - /// `AggregateCoverage::definition_id`. + /// The plan-scope content id of what the view computes. pub fn definition_id(&self) -> &str { &self.definition_id } @@ -601,7 +606,7 @@ impl PhysicalASAPDAG { self.operator_nodes()? }; let mut derived = Vec::new(); - let mut coverage_by_node: HashMap = HashMap::new(); + let mut coverage_by_node: HashMap = HashMap::new(); for &id in index.children_first.iter().filter(|id| views.contains(id)) { let node = index.nodes[&id]; let (inputs, scans) = index.inputs_of_view(id, &views)?; @@ -642,10 +647,7 @@ impl PhysicalASAPDAG { } else { MaterializedViewRefresh::OnFirstRead }, - coverage: ViewCoverage { - definition_id: coverage.definition_id(), - selection: coverage.selection, - }, + coverage, inputs, retention_ms: node.retention_ms.unwrap_or(0), }; @@ -926,16 +928,18 @@ impl<'a> ViewIndex<'a> { } /// The coverage of view `id` (IR `ir`), refreshed from `inputs`: the -/// coverage its memo group stores, which the realized plan will carry. +/// selection its memo group's `aggregate_coverage` stores, which the +/// realized plan will carry, and the content id of that coverage's +/// definition expanded to the `Scan`s ([`ViewCoverage::definition_id`]). /// Coverage is derived only in the memo, never on node trees. fn coverage_of( id: PhysicalASAPNodeId, inputs: &MaterializedViewInputs, ir: &HashMap>, - coverage_by_node: &HashMap, -) -> Result { + coverage_by_node: &HashMap, +) -> Result { let _ = (id, inputs, ir, coverage_by_node); todo!( - "#1207 PR 6: the view's coverage from its memo group's GroupProperties::aggregate_coverage" + "#1207 PR 6: the view's coverage from its memo group's aggregate_coverage, with definition_id the content id of the expanded definition" ) } diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index fd1cd19cc..b06224c12 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -120,23 +120,6 @@ pub enum CoverageError { RegroupedBuild { reason: String }, } -impl AggregateCoverage { - /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, - /// of its input group and unlifted operators. Two equal definitions - /// (the equality a merge requires) of one memo get equal ids, so a - /// summary store can key states by this id and a concrete selection. - pub(crate) fn definition_id(&self) -> String { - const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; - const PRIME: u128 = 0x0000000001000000000000000000013b; - let mut hash = OFFSET; - for byte in format!("{:?}", self.definition).bytes() { - hash ^= u128::from(byte); - hash = hash.wrapping_mul(PRIME); - } - format!("{hash:032x}") - } -} - /// Coverage of what a memo group computes with `operator` over its /// input groups: a `SummaryAgg`, an exact `Aggregate`, a /// `SummaryTumblingWindowChain` (a build over its input, the timestamp @@ -375,7 +358,7 @@ fn regrouped( reason: error.to_string(), } })?; - // A direct build's definition, so its definition id is the same. + // A direct build's definition, so it is `==` to one. let mut definition = merged.definition.clone(); definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { child: None, diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 5f0605a84..60833bac8 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -9,18 +9,12 @@ pub mod memo_fixtures; use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; use crate::plan::materialized_view::RawDataWindow; -use crate::stages::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::{ASAPOp, NonASAPOp}; use crate::ir::{OperatorNode, SchemaDerivationError}; use std::rc::Rc; -/// `AggregateCoverage::definition_id`. -pub fn definition_id(coverage: &AggregateCoverage) -> String { - coverage.definition_id() -} - /// `Operator::output_schema`. pub fn output_schema(operator: &Operator) -> Result { operator.output_schema() diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index 457dc0dbf..31b919290 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -26,7 +26,6 @@ use asap_types::stages::{ MemoGroupId, OriginOfCandidateForMemoGroup, }; use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; -use asap_types::test_support::definition_id; use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; use asap_types::workload::DataWorkload; @@ -183,7 +182,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { assert_eq!(selection.columns()[&ts()], last_minute()); } -/// G7 has G4's definition and id and a disjoint selection, so +/// G7 has G4's definition (`==`) and a disjoint selection, so /// `SummaryMerge(G4, G7)` is a valid group covering job ∈ {api, web}. #[test] fn states_with_equal_definitions_and_disjoint_selections_merge() { @@ -195,10 +194,6 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { coverage(&memo, g4).definition(), coverage(&memo, g7).definition() ); - assert_eq!( - definition_id(coverage(&memo, g4)), - definition_id(coverage(&memo, g7)) - ); let merged = memo.find_or_add_group(merge(vec![g4, g7])).unwrap(); let coverage = coverage(&memo, merged); assert_eq!(coverage.definition().input_group, Some(g1)); diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 3eb03b8c5..34924e89c 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -540,7 +540,7 @@ pub struct MaterializedViewRequest<'a> { | | `read_raw_data` | `read_materialized_view` | | --- | --- | --- | -| Keyed by | `Source` | `MaterializedView` (its coverage: `definition_id` and selection) | +| Keyed by | `Source` | `MaterializedView` (its coverage: the content id `definition_id` and selection) | | Time range | optional, a hint | always given, exact | | Predicates | a hint | none | | Executor re-filters | yes | no | @@ -663,7 +663,13 @@ Query at T = 12:05 over (12:00, 12:05]: ``` The deployment matches a view to stored states by its coverage -(`definition_id` and selection). At execution time it checks their actual +(`definition_id` and selection). Identity has two scopes: the memo compares +coverages by `AggregateDefinition` `==` (its definitions hold memo group ids) +and needs no id; the plan carries `definition_id`, a content id of what the +view computes: its definition with every input group expanded recursively to +its expression down to the `Scan`s, with no group or node ids, canonically +serialized and hashed. It is equal across plans and processes that compute the +same thing and different when they do not. At execution time it checks their actual coverage, revisions and readiness. A compiled plan cannot establish future readiness. The deployment does not replace `NativeKllMerge`, choose another sketch, or maintain different windows: such changes require replanning. @@ -682,7 +688,8 @@ sketch, or maintain different windows: such changes require replanning. | Where does retention live? | On the materialized-view node of the exported DAG | — | Mapping the materialized-views extension's terms: its `query()` is the sub-DAG -under the view's node, and `coverage.definition_id` identifies that sub-DAG. +under the view's node, and `coverage.definition_id`, the content id of that +sub-DAG's expanded definition, identifies it. Its readiness is the summary store's status, which the deployment owns. Its `mv_dependencies` build graph is `MaterializedView.inputs`. @@ -741,7 +748,7 @@ operator/runtime fixtures: | Test | Contract exercised | | --- | --- | | `kll_tumbling_window_execution::five_tumbling_windows_roundtrip_and_shared_merge_runs_once` | One-minute tumbling window refreshes from raw data → five materialized view slots → shared native merge → p50/p99; counts every sample once, checks adjacent aligned windows and instruments one merge start per run | -| `kll_tumbling_window_execution::a_store_keyed_by_definition_and_time_range_answers_the_tumbling_windows` | A store that finds states only by `(definition_id, time range)` answers the five-tumbling-window query: the tumbling windows share one definition and request the five ranges that tile the window | +| `kll_tumbling_window_execution::a_store_keyed_by_definition_and_time_range_answers_the_tumbling_windows` | A store that finds states only by `(definition_id, time range)` (the content id is an opaque string to the store) answers the five-tumbling-window query: the tumbling windows share one definition and request the five ranges that tile the window | | `runtime_install::tumbling_windows_of_one_refresh_window_read_each_raw_window_once` | A KLL chain and a DDSketch chain of 10 s tumbling windows over one `Scan` are one refresh group: each tumbling window window is read from raw data once, in one ingestion run that builds both tumbling windows; each state equals that of a run of its view alone, and both queries answer as the query-time reference | | `kll_tumbling_window_execution::tumbling_windows_reject_parameters_schema_and_missing_binding` | Parameter relabelling, incompatible schemas and missing views fail explicitly | | `precompute_candidates::grouped_rate_can_be_materialized_before_or_after_grouped_sum` | Stored Rate state and stored Sum state answer like query time; summing counters before Rate does not | diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 179eb6f6e..f96043489 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -266,8 +266,6 @@ impl AggregateCoverage { pub(crate) fn derive_from_inputs( operator: &Operator>, ) -> Result; - /// A hash of the definition: equal definitions of one memo get equal ids. - pub(crate) fn definition_id(&self) -> String; } ``` @@ -299,7 +297,7 @@ G4.aggregate_coverage = AggregateCoverage { } ``` -**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) (`==`); its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. **Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: @@ -320,7 +318,7 @@ Its definition differs from G4's, so G10 cannot merge with G4, even though both - `input_group`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). - `unlifted_operators`: what the state computes over `input_group`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. - `selection`: which output rows of that computation the state took, as a union of boxes. -- `definition_id()`: a hash of the definition, the key a summary store files states under together with a concrete selection. +- Identity, in two scopes. In the memo, coverages compare by `AggregateDefinition` `==` (it holds `MemoGroupId`s); nothing in the memo needs an id. The plan carries `ViewCoverage::definition_id`, a content id: the definition with every input group expanded recursively to its expression down to the `Scan`s, with no group or node ids, canonically serialized and hashed. It is equal across plans and processes that compute the same thing and different when they do not, and it is the key a summary store files states under together with a concrete selection. ##### The definition @@ -1454,7 +1452,6 @@ impl AggregateCoverage { pub(crate) fn derive_from_inputs( operator: &Operator>, ) -> Result; - pub(crate) fn definition_id(&self) -> String; } ``` From 288cd70d67ba0c7941462c1b37ff5b144eccfd8e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:32:10 +0000 Subject: [PATCH 10/12] Sign bit for ValueSign; point_counts_may_flip_sign; view coverage on the plan node - ValueSign is the IEEE 754 sign bit: NonNegative means every non-NaN value has its sign bit clear, NonPositive set. sign_of_number reads the sign bit (-0.0 is NonPositive, NaN Unknown), so `x / -0` over a non-negative x is NonPositive, no longer NonNegative. - SummaryTypeCapability::point_counts_may_flip_sign (was estimates_may_flip_sign) is pub(crate) and an exhaustive match over SketchAlgorithm. - A readout's sign rule takes the state column's type (readout_signs's value(sign, dtype)); the separate state-field lookup is gone. - PhysicalASAPDAGNode::view_coverage carries a materialized view's coverage into the plan; the views code reads it. coverage_of, the IR it needed and coverage_by_node are deleted; operator_nodes is test-only. Stage 3's todo names where it is filled (the content id of the expanded definition). Serde default, so plan files and fixtures are unchanged. - The content id is defined once, on ViewCoverage::definition_id; the design docs link to it. - Test rows without "(was Unknown)" notes. --- crates/executor/tests/current_timestamp.rs | 1 + crates/executor/tests/hydra_cms_execution.rs | 1 + .../tests/keyed_summary_row_weight.rs | 1 + crates/executor/tests/physical_dag.rs | 5 ++ crates/executor/tests/physical_semantics.rs | 3 + crates/executor/tests/plan_properties.rs | 1 + .../executor/tests/precompute_population.rs | 4 ++ crates/executor/tests/promql_binary.rs | 2 + crates/executor/tests/raw_scan.rs | 3 + .../stored_tumbling_window_memory_limit.rs | 1 + crates/executor/tests/summary_projection.rs | 4 ++ crates/executor/tests/univmon_execution.rs | 1 + crates/stage3-plan-selection/src/lib.rs | 5 +- crates/types/src/plan/materialized_view.rs | 33 +-------- crates/types/src/plan/physical_asap_dag.rs | 25 +++++-- .../src/stages/logical/column_properties.rs | 68 +++++++++++-------- crates/types/src/summary_type/capability.rs | 56 ++++++++++----- crates/types/tests/memo_group_properties.rs | 16 +++-- crates/types/tests/physical_export.rs | 2 + .../physical-planning-and-deployment.md | 14 ++-- .../proposals/asap-primitive-schema.md | 2 +- 21 files changed, 150 insertions(+), 98 deletions(-) diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index 415b7c0f9..1d1a2eeab 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -71,6 +71,7 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 628737eab..b7ca69528 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -61,6 +61,7 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 8ceb8bdb2..0733da866 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -164,6 +164,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let build = compile_node(&node, std::slice::from_ref(&input)).unwrap(); let wire = serde_json::to_value(&build).unwrap(); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 9bc4f7fdd..517067f4e 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -537,6 +537,7 @@ fn bind_physical_asap_dag_before_execution() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let mut dag = PhysicalASAPDAG { nodes: vec![ @@ -699,6 +700,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer, role, schema: &asap_executor::values::SchemaRef| { PhysicalASAPDAGEdge::new( @@ -1120,6 +1122,7 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -1222,6 +1225,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { right: 1, }), selected_physical_candidate: None, + view_coverage: None, }; let join = if certified { Operator::certified_semi_join(schema.clone(), schema.clone(), vec![(0, 0)]).unwrap() @@ -1329,6 +1333,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { rhs: 1, }), selected_physical_candidate: None, + view_coverage: None, }; let binary = compile_node(&node, &[input.clone(), input.clone()]).unwrap(); for (right, expected) in [ diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index eb395598c..a9b0ccc02 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -391,6 +391,7 @@ fn global_extrema_bind_with_planner_derived_schema() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let operator = compile_node(&node, std::slice::from_ref(&input)) .expect("global extremum should bind to its Planner schema"); @@ -935,6 +936,7 @@ fn planner_node( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } @@ -1168,6 +1170,7 @@ fn exact_aggregate_node( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 6f1aa506d..64f64962f 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -55,6 +55,7 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; PhysicalASAPDAG { nodes: vec![ diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 2ad42fbf7..e9af0acad 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -53,6 +53,7 @@ fn view_over_samples(family: FieldDataType, scan: usize) -> [PhysicalASAPDAGNode guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; [ PhysicalASAPDAGNode { @@ -141,6 +142,7 @@ fn try_state_dag( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); } let read_id = nodes.len(); @@ -156,6 +158,7 @@ fn try_state_dag( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); if let Some(target) = target { nodes.push(PhysicalASAPDAGNode { @@ -176,6 +179,7 @@ fn try_state_dag( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); } let root = nodes.last().unwrap().id; diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 99d260de4..29c23131b 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -87,6 +87,7 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; compile_node(&node, &[schema.clone(), schema]).unwrap() } @@ -428,6 +429,7 @@ fn stored_series_evaluations_support_filters_and_sets() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }) .collect::>(); let edges = [ diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 9edc47cf9..830b799be 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -91,6 +91,7 @@ fn plan( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -513,6 +514,7 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -848,6 +850,7 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } }; dag.nodes.push(node( diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index cc8019aa3..f5391cf28 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -323,6 +323,7 @@ fn merge_over_a_stored_tumbling_window( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let nodes = vec![ node( diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 3e14c8b2e..1ab25998e 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -58,6 +58,7 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } @@ -130,6 +131,7 @@ fn summary_projection_survives_recovery() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }, raw_node(2, scan()), raw_node( @@ -160,6 +162,7 @@ fn summary_projection_survives_recovery() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }, ], edges: vec![ @@ -269,6 +272,7 @@ fn summary_merge_without_inputs_is_rejected() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 2ff8822dd..341977372 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -185,6 +185,7 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let build = compile_node(&node, std::slice::from_ref(&input))?; let batch = Batch::try_new( diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 2410b88a5..607d9386d 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -28,7 +28,8 @@ pub use report::{ /// Stage 3, plan selection: select one plan among `physical`'s candidates /// (module documentation) under `cost_inputs`, and build it: one /// `PhysicalASAPDAG` node per selected execution, its inputs from the -/// selected edges, each node's `selected_physical_candidate` set. +/// selected edges, each node's `selected_physical_candidate` set and each +/// materialized view node's `view_coverage` from its memo group. /// `entries[q]` is query `q`'s workload entry; `data` the data workload /// with its bounds. /// @@ -44,5 +45,5 @@ pub fn select_plan( .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; let _ = (physical, entries, data); - todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution") + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage from its memo group's aggregate_coverage, with definition_id the content id of the expanded definition") } diff --git a/crates/types/src/plan/materialized_view.rs b/crates/types/src/plan/materialized_view.rs index 33fb6f22e..2a87062fb 100644 --- a/crates/types/src/plan/materialized_view.rs +++ b/crates/types/src/plan/materialized_view.rs @@ -64,7 +64,6 @@ //! it once. use std::collections::{HashMap, HashSet}; -use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -74,11 +73,10 @@ use super::physical_asap_dag::{ }; use crate::ir::operator::arguments::Source; use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::selection::SelectionBox; -use crate::stages::CoverageError; use crate::stages::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is @@ -599,14 +597,7 @@ impl PhysicalASAPDAG { .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| node.id) .collect(); - // The IR of each node, which coverage is derived from. - let ir = if views.is_empty() { - HashMap::new() - } else { - self.operator_nodes()? - }; let mut derived = Vec::new(); - let mut coverage_by_node: HashMap = HashMap::new(); for &id in index.children_first.iter().filter(|id| views.contains(id)) { let node = index.nodes[&id]; let (inputs, scans) = index.inputs_of_view(id, &views)?; @@ -633,13 +624,12 @@ impl PhysicalASAPDAG { ); } } - let coverage = coverage_of(id, &inputs, &ir, &coverage_by_node).map_err(|error| { + let coverage = node.view_coverage.clone().ok_or_else(|| { InstallableDagError::ViewWithoutCoverage { node_id: id, - reason: error.to_string(), + reason: "the plan node carries no view_coverage".into(), } })?; - coverage_by_node.insert(id, coverage.clone()); let view = MaterializedView { node_id: id, refresh: if refreshed_at_ingestion { @@ -926,20 +916,3 @@ impl<'a> ViewIndex<'a> { Ok((inputs, scans)) } } - -/// The coverage of view `id` (IR `ir`), refreshed from `inputs`: the -/// selection its memo group's `aggregate_coverage` stores, which the -/// realized plan will carry, and the content id of that coverage's -/// definition expanded to the `Scan`s ([`ViewCoverage::definition_id`]). -/// Coverage is derived only in the memo, never on node trees. -fn coverage_of( - id: PhysicalASAPNodeId, - inputs: &MaterializedViewInputs, - ir: &HashMap>, - coverage_by_node: &HashMap, -) -> Result { - let _ = (id, inputs, ir, coverage_by_node); - todo!( - "#1207 PR 6: the view's coverage from its memo group's aggregate_coverage, with definition_id the content id of the expanded definition" - ) -} diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index e4ff4b4fd..9d108ade6 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -9,14 +9,18 @@ //! (#509 Assumption 8): it does not change lifetime or timing. use std::collections::{HashMap, HashSet}; +#[cfg(any(test, feature = "test-support"))] use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; +use super::materialized_view::ViewCoverage; use crate::ir::flat::NodeId; use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::node::Operator; +#[cfg(any(test, feature = "test-support"))] +use crate::ir::operator::node::OperatorNode; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::scalar::ScalarExpr; @@ -95,6 +99,14 @@ pub struct PhysicalASAPDAGNode { /// built by a custom pass. #[serde(skip)] pub selected_physical_candidate: Option, + /// For a materialized view (lifetime `AcrossEvaluations`), what it + /// stores: its memo group's aggregate coverage, carried into the plan + /// as a [`ViewCoverage`] (the selection, and the plan-scope content id + /// of the definition, [`ViewCoverage::definition_id`]). Set when the + /// plan is built from the memo, where the node's group is known; `None` + /// on every other node. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub view_coverage: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -256,11 +268,11 @@ impl PhysicalASAPDAG { } /// Every node of this DAG as IR, by node id, with its exported - /// guarantee (an operator node has no physical properties): what a caller derives - /// IR properties of a plan file's node from, such as its - /// [`AggregateCoverage`](crate::stages::AggregateCoverage). - /// Shared inputs stay shared. Validates the DAG first. - pub(crate) fn operator_nodes( + /// guarantee (an operator node has no physical properties): what + /// [`Self::query_roots`] reads. Shared inputs stay shared. Validates the + /// DAG first. + #[cfg(any(test, feature = "test-support"))] + fn operator_nodes( &self, ) -> Result>, PhysicalASAPDAGValidationError> { self.validate()?; @@ -607,6 +619,7 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( guarantee: flat_node.guarantee, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); } let dag = PhysicalASAPDAG { diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index 0a24b7c06..a60c88cfe 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -25,12 +25,18 @@ use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; /// input groups like lineage (concepts doc, section 3, "Value sign"); a /// `Scan`'s are declared per column in the data workload's source facts. /// Error contracts that hold only for values of one sign read it. +/// +/// A sign is the IEEE 754 sign bit, so `-0.0` is `NonPositive` and `+0.0` +/// `NonNegative`; with it the rules are exact: a product or quotient has +/// the XOR of its operands' sign bits, a sum of two values with one sign +/// bit keeps it, a negation flips it and `abs` clears it. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] pub enum ValueSign { - /// Every non-NaN value is ≥ 0. + /// Every non-NaN value has its sign bit clear (`+0.0`, positive, + /// `+inf`). NonNegative, - /// Every non-NaN value is ≤ 0. + /// Every non-NaN value has its sign bit set (`-0.0`, negative, `-inf`). NonPositive, /// Neither is known. Unknown, @@ -243,26 +249,20 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { SummaryEstimate { summary_input, query, - } => { - // The summary type of the state read, for whether its counts keep - // the weights' sign. - let flips = summary_input - .schema() - .fields - .iter() - .find(|field| !field.is_plain()) - .is_some_and(|state| SummaryTypeCapability::estimates_may_flip_sign(&state.dtype)); - readout_signs(summary_input, output, |state| match query { - SketchStatistic::Cardinality - | SketchStatistic::FrequencyL2 - | SketchStatistic::FrequencyEntropy => NonNegative, - SketchStatistic::Quantile { q } => quantile_sign(*q, state), - SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } if flips => { - Unknown - } - SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } => state, - }) - } + } => readout_signs(summary_input, output, |state, summary_type| match query { + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy => NonNegative, + SketchStatistic::Quantile { q } => quantile_sign(*q, state), + // A count read from a state whose counts may flip the weights' + // sign has none. + SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } + if SummaryTypeCapability::point_counts_may_flip_sign(summary_type) => + { + Unknown + } + SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } => state, + }), FinalizeExactAccumulator { child } => { let counts = child.schema().fields.iter().any(|field| { matches!( @@ -273,7 +273,7 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { readout_signs( child, output, - |state| if counts { NonNegative } else { state }, + |state, _| if counts { NonNegative } else { state }, ) } MaintainPopulation { child, .. } => signs_of(child), @@ -417,13 +417,14 @@ fn state_signs( } /// A readout of summary state: the state's plain columns keep their sign -/// in place, and the value read out has `value(the state's recorded sign)`. +/// in place, and the value read out has `value(the state's recorded sign, +/// the state column's type)`. /// `Unknown` everywhere when the output does not keep the state's layout (a /// top-k's rows). fn readout_signs( state: &Input<'_>, output: &Schema, - value: impl Fn(ValueSign) -> ValueSign, + value: impl Fn(ValueSign, &FieldDataType) -> ValueSign, ) -> Vec { let input = state.schema(); if input.fields.len() != output.fields.len() { @@ -432,7 +433,13 @@ fn readout_signs( signs_of(state) .into_iter() .zip(&input.fields) - .map(|(sign, field)| if field.is_plain() { sign } else { value(sign) }) + .map(|(sign, field)| { + if field.is_plain() { + sign + } else { + value(sign, &field.dtype) + } + }) .collect() } @@ -495,12 +502,13 @@ fn flipped(sign: ValueSign) -> ValueSign { } } +/// The sign of a number: its sign bit; `Unknown` for NaN. fn sign_of_number(value: f64) -> ValueSign { - if value >= 0.0 { - NonNegative - } else if value < 0.0 { + if value.is_nan() { + Unknown + } else if value.is_sign_negative() { NonPositive } else { - Unknown + NonNegative } } diff --git a/crates/types/src/summary_type/capability.rs b/crates/types/src/summary_type/capability.rs index 63f873244..b615192e9 100644 --- a/crates/types/src/summary_type/capability.rs +++ b/crates/types/src/summary_type/capability.rs @@ -114,24 +114,46 @@ impl SummaryTypeCapability { /// may have a sign the weights it was updated with do not: a median of /// signed counters (Count Sketch, UnivMon and their variants) can be /// negative over non-negative weights; an overestimating counter - /// (Count-Min) or an exact count cannot. - pub fn estimates_may_flip_sign(summary_type: &FieldDataType) -> bool { + /// (Count-Min) or an exact count cannot. Every algorithm is listed, so + /// one `asap_sketchlib` adds fails to compile here until it is placed. + /// `asap_sketchlib`'s `experimental` algorithms are not listed: the + /// planner does not build it with that feature, as `parameter_list` + /// does not. + pub(crate) fn point_counts_may_flip_sign(summary_type: &FieldDataType) -> bool { use SketchAlgorithm as A; - matches!( - summary_type, - FieldDataType::Sketch( - A::CountSketch - | A::CountSketchFold - | A::CountSketchWithHeap - | A::CountSketchWithL2Norm - | A::CountSketchWithSpaceSaving - | A::HydraCountSketch - | A::UnivMon - | A::UnivMonQ - | A::UnivMonTwoTier - | A::HydraUnivMon - ) - ) + let FieldDataType::Sketch(algorithm) = summary_type else { + return false; + }; + match algorithm { + A::CountSketch + | A::CountSketchFold + | A::CountSketchWithHeap + | A::CountSketchWithL2Norm + | A::CountSketchWithSpaceSaving + | A::HydraCountSketch + | A::UnivMon + | A::UnivMonQ + | A::UnivMonTwoTier + | A::HydraUnivMon => true, + A::Cms + | A::CmsWithHeap + | A::CmsWithSpaceSaving + | A::CmsFold + | A::HydraCms + | A::Hll + | A::HllHip + | A::Kmv + | A::Kll + | A::KllDynamic + | A::HydraKll + | A::HydraHll + | A::DDSketch + | A::SpaceSaving + | A::Elastic + | A::Coco + | A::Bloom + | A::SetAggregator => false, + } } /// Whether `summary_type` is a Hydra algorithm: one structure shared by diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index db6e69082..8eaec83de 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -665,10 +665,10 @@ fn arithmetic(op: ArithmeticOpKind, lhs: MemoGroupId, rhs: MemoGroupId) -> Opera /// |---|---|---| /// | G3 | `quantile_over_time(0.9, G2)`, G2 = `latency[1h]` | `NonNegative` | /// | G4 | `quantile_over_time(0.5, G2)` | `NonNegative` | -/// | G5 | `G3 / G4` | `NonNegative` (was `Unknown`) | +/// | G5 | `G3 / G4` | `NonNegative` | /// | G6 | `G3 + G4` | `NonNegative` | /// | G7 | `G3 - G4` | `Unknown` | -/// | G8 | `vector(1000)` | `NonNegative` (was `Unknown`) | +/// | G8 | `vector(1000)` | `NonNegative` | /// | G9 | `vector(-1)` | `NonPositive` | /// | G10 | `G3 * G8` | `NonNegative` | /// | G11 | `G3 * G9` | `NonPositive` | @@ -729,11 +729,12 @@ fn vector_arithmetic_follows_its_operands_signs() { /// /// | Group | Expression | value sign | /// |---|---|---| -/// | G2 | `G1 / 1000`, G1 = `Scan latency` (PromQL) | `NonNegative` (was `Unknown`) | -/// | G3 | `G1 * 8` (PromQL) | `NonNegative` (was `Unknown`) | -/// | G4 | `SELECT value * 8 FROM G1` (SQL) | `NonNegative` (was `Unknown`) | +/// | G2 | `G1 / 1000`, G1 = `Scan latency` (PromQL) | `NonNegative` | +/// | G3 | `G1 * 8` (PromQL) | `NonNegative` | +/// | G4 | `SELECT value * 8 FROM G1` (SQL) | `NonNegative` | /// | G5 | `G1 - G1` | `Unknown` | /// | G6 | `G1 * -8` | `NonPositive` | +/// | G7 | `G1 / -0` | `NonPositive` (`-0` has its sign bit set: the quotient is `-inf`, `-0` or NaN) | #[test] fn scalar_arithmetic_follows_its_operands_signs() { let mut memo = memo(); @@ -776,12 +777,17 @@ fn scalar_arithmetic_follows_its_operands_signs() { &mut memo, map_value(expr(ArithmeticOpKind::Mul, number(-8.0), promql), g1), ); + let g7 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Div, number(-0.0), promql), g1), + ); for (group, value) in [ (g2, NonNegative), (g3, NonNegative), (g4, NonNegative), (g5, Unknown), (g6, NonPositive), + (g7, NonPositive), ] { assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{group:?}"); } diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 828ee1993..23d9e93a8 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -298,6 +298,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }) .collect(); let inputs: Vec<_> = (0..count).chain([0, 0]).collect(); @@ -313,6 +314,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); let mut dag = PhysicalASAPDAG { nodes, diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 34924e89c..a0d8e8ea8 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -665,11 +665,10 @@ Query at T = 12:05 over (12:00, 12:05]: The deployment matches a view to stored states by its coverage (`definition_id` and selection). Identity has two scopes: the memo compares coverages by `AggregateDefinition` `==` (its definitions hold memo group ids) -and needs no id; the plan carries `definition_id`, a content id of what the -view computes: its definition with every input group expanded recursively to -its expression down to the `Scan`s, with no group or node ids, canonically -serialized and hashed. It is equal across plans and processes that compute the -same thing and different when they do not. At execution time it checks their actual +and needs no id; the plan carries the view's content id on its node +(`PhysicalASAPDAGNode::view_coverage`), defined at +[`ViewCoverage::definition_id`](../../crates/types/src/plan/materialized_view.rs). +At execution time it checks their actual coverage, revisions and readiness. A compiled plan cannot establish future readiness. The deployment does not replace `NativeKllMerge`, choose another sketch, or maintain different windows: such changes require replanning. @@ -688,8 +687,9 @@ sketch, or maintain different windows: such changes require replanning. | Where does retention live? | On the materialized-view node of the exported DAG | — | Mapping the materialized-views extension's terms: its `query()` is the sub-DAG -under the view's node, and `coverage.definition_id`, the content id of that -sub-DAG's expanded definition, identifies it. +under the view's node, and `coverage.definition_id` (the content id, +[`ViewCoverage::definition_id`](../../crates/types/src/plan/materialized_view.rs)) +identifies it. Its readiness is the summary store's status, which the deployment owns. Its `mv_dependencies` build graph is `MaterializedView.inputs`. diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index f96043489..e80626db9 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -318,7 +318,7 @@ Its definition differs from G4's, so G10 cannot merge with G4, even though both - `input_group`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). - `unlifted_operators`: what the state computes over `input_group`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. - `selection`: which output rows of that computation the state took, as a union of boxes. -- Identity, in two scopes. In the memo, coverages compare by `AggregateDefinition` `==` (it holds `MemoGroupId`s); nothing in the memo needs an id. The plan carries `ViewCoverage::definition_id`, a content id: the definition with every input group expanded recursively to its expression down to the `Scan`s, with no group or node ids, canonically serialized and hashed. It is equal across plans and processes that compute the same thing and different when they do not, and it is the key a summary store files states under together with a concrete selection. +- Identity, in two scopes. In the memo, coverages compare by `AggregateDefinition` `==` (it holds `MemoGroupId`s); nothing in the memo needs an id. The plan carries the view's content id, [`ViewCoverage::definition_id`](../../../crates/types/src/plan/materialized_view.rs) (defined there), on the view's plan node (`PhysicalASAPDAGNode::view_coverage`); a summary store files states under it together with a concrete selection. ##### The definition From 0de33fd94698fe6b36d4cf4b35830ea765a43c3a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:52:52 +0000 Subject: [PATCH 11/12] Memo query lookups as methods; Operator::scanned_sources; coverage wording; shared memo test helpers - root_groups and queries_rooted_at are Memo methods (root_groups(&self), queries_rooted_at(&self, group)) instead of free functions over &[MemoQuery]; the planner CLI calls the method. - sources_scanned_by (ir/operator/input.rs) is Operator::scanned_sources in ir/operator/node.rs, with the other rules generic over the input. - "summary coverage" wording left over from the rename now says aggregate coverage (crate docs, the merge-validation comment, a SQL lowering test comment, the architecture doc and the design doc's section 6.6 row). - aggregate, aggregate_where and readout are in test_support::memo_fixtures, used by the memo unit tests and memo_group_properties instead of their own copies. --- crates/devtools/src/bin/asap_planner.rs | 4 +- crates/frontend-sql/tests/sql_lowering.rs | 2 +- crates/types/src/ir/operator/asap.rs | 5 +- crates/types/src/ir/operator/input.rs | 19 +------ crates/types/src/ir/operator/node.rs | 17 +++++- crates/types/src/lib.rs | 3 +- crates/types/src/stages/logical/group.rs | 4 +- crates/types/src/stages/logical/memo/mod.rs | 30 +++++----- crates/types/src/stages/logical/memo/tests.rs | 56 ++++++++++--------- crates/types/src/stages/logical/mod.rs | 2 +- crates/types/src/stages/mod.rs | 10 ++-- .../types/src/test_support/memo_fixtures.rs | 38 ++++++++++++- crates/types/tests/memo_group_properties.rs | 26 +-------- .../proposals/asap-primitive-schema.md | 2 +- .../asap-aware-mapping-architecture.md | 2 +- 15 files changed, 122 insertions(+), 98 deletions(-) diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index a802820f0..7ddbe3ba4 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -109,8 +109,8 @@ use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; use asap_types::ir::QueryRoot; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::stages::PhysicalASAPMemoCandidates; -use asap_types::stages::{queries_rooted_at, MemoGroup, MemoGroupId}; use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; +use asap_types::stages::{MemoGroup, MemoGroupId}; use asap_types::workload::{ DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, @@ -757,7 +757,7 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) "children": group.expression.children().into_iter().map(|g| g.0).collect::>(), "parents": ids(group.parent_groups()), "queries": group.reached_by_queries(), - "roots_of": queries_rooted_at(memo.queries(), MemoGroupId(id as u32)), + "roots_of": memo.queries_rooted_at(MemoGroupId(id as u32)), "node": node, "candidate_count": candidates.len(), "candidate_count_by_origin": by_origin, diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 2a83de38e..4ae70b9f9 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -3203,7 +3203,7 @@ async fn trailing_window_on_an_aliased_table_and_under_asap_rate() { } // An absolute timestamp filter compares the time column with a `Timestamp` -// literal (ms since the epoch), so summary coverage lifts it into an +// literal (ms since the epoch), so aggregate coverage lifts it into an // interval on the timestamp column: the cast of the string folds, and the // cast of the Timestamp column to DataFusion's nanosecond unit is dropped. #[tokio::test] diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 037c6d57f..1793cbbd6 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -514,8 +514,9 @@ impl ASAPOp { } } // Equal schemas cannot tell a KLL over `latency` from one over - // `size`, nor prove the inputs disjoint; summary coverage (#646) - // decides whether a structurally valid merge is semantically valid. + // `size`, nor prove the inputs disjoint; aggregate coverage (in + // the memo) decides whether a structurally valid merge is + // semantically valid. Ok(()) } SummaryEstimate { diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index 2aa3b9755..4c53e4c37 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -24,24 +24,11 @@ pub trait OperatorInput: Sized + Clone { /// `None` for any other input. fn aggregate_coverage(&self) -> Option<&AggregateCoverage>; /// The sources the `Scan`s at or below the input read - /// (`sources_scanned_by` of its operator): a memo group's stored + /// (`Operator::scanned_sources` of its operator): a memo group's stored /// `GroupProperties::scanned_sources`, recomputed for a node tree. fn scanned_sources(&self) -> BTreeSet; } -/// The sources the `Scan`s at or below `operator` read: a `Scan`'s own, -/// else the union over its inputs' ([`OperatorInput::scanned_sources`]). -pub(crate) fn sources_scanned_by(operator: &Operator) -> BTreeSet { - match operator { - Operator::NonASAP(NonASAPOp::Scan { source, .. }) => BTreeSet::from([source.clone()]), - operator => operator - .children() - .into_iter() - .flat_map(OperatorInput::scanned_sources) - .collect(), - } -} - /// A front-end node tree. Node trees carry no aggregate coverage. impl OperatorInput for Rc { fn schema(&self) -> &Schema { @@ -61,7 +48,7 @@ impl OperatorInput for Rc { } fn scanned_sources(&self) -> BTreeSet { - sources_scanned_by(&self.operator) + self.operator.scanned_sources() } } @@ -142,7 +129,7 @@ impl OperatorInput for InputOrTemporaryBuild { fn scanned_sources(&self) -> BTreeSet { match self { Self::Input(input) => input.scanned_sources(), - Self::TemporaryBuild(_) => sources_scanned_by(&self.operator()), + Self::TemporaryBuild(_) => self.operator().scanned_sources(), } } } diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 7551f4d6c..72bc8ad8f 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -5,11 +5,12 @@ //! each physical candidate holds its own //! ([`crate::stages::PhysicalCandidate::properties`]). -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; use std::rc::Rc; use serde::{Deserialize, Serialize}; +use super::arguments::Source; use super::asap::ASAPOp; use super::input::OperatorInput; use super::non_asap::NonASAPOp; @@ -65,6 +66,20 @@ impl Operator { /// The derivation rules, one set for every kind of input /// ([`OperatorInput`]): front-end node trees and memo groups. impl Operator { + /// The sources the `Scan`s at or below this operator read: a `Scan`'s + /// own, else the union over its inputs' + /// ([`OperatorInput::scanned_sources`]). + pub(crate) fn scanned_sources(&self) -> BTreeSet { + match self { + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => BTreeSet::from([source.clone()]), + operator => operator + .children() + .into_iter() + .flat_map(OperatorInput::scanned_sources) + .collect(), + } + } + pub(crate) fn output_schema(&self) -> Result { match self { Operator::NonASAP(op) => op.output_schema(), diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 6c2659b09..d805699f6 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -6,7 +6,8 @@ //! [`ir::operator`] (§1 operators and their argument types), //! [`ir::scalar`] (§2.2 scalar expressions), [`ir::schema`] (§2.1 the //! per-edge schema) and [`ir::properties`] (§2.3 execution timing and -//! summary coverage), and the flat DAG form ([`ir::flat`]). +//! the selection algebra aggregate coverage is stated in), and the flat +//! DAG form ([`ir::flat`]). //! - [`summary_type`] — each summary type's parameters and predefined //! parameter lists, the statistics read from it and its capabilities. //! - [`accuracy`] — accuracy targets, the error metrics they are stated in, diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 1d1765962..2b4f410d6 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -6,7 +6,7 @@ use std::collections::BTreeSet; use super::aggregate_coverage::{aggregate_coverage, AggregateCoverage}; use super::column_properties::{column_signs, ValueSign}; use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; -use crate::ir::operator::input::{sources_scanned_by, OperatorInput}; +use crate::ir::operator::input::OperatorInput; use crate::ir::schema::Schema; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; @@ -120,7 +120,7 @@ pub(super) fn derive_group_properties( result_kind: operator.output_kind(), aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), - scanned_sources: sources_scanned_by(&operator), + scanned_sources: operator.scanned_sources(), }) } diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 331f69afc..647cfa18a 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -316,21 +316,23 @@ fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { } } -/// The groups some query's root reads, ascending: the groups of the -/// `queries`' roots that are a query's result or a scalar query's operand. -pub fn root_groups(queries: &[MemoQuery]) -> BTreeSet { - queries - .iter() - .flat_map(|query| query.root.operator_refs()) - .copied() - .collect() -} +impl Memo { + /// The groups some query's root reads, ascending: the groups of the + /// queries' roots that are a query's result or a scalar query's operand. + pub fn root_groups(&self) -> BTreeSet { + self.queries + .iter() + .flat_map(|query| query.root.operator_refs()) + .copied() + .collect() + } -/// The queries whose root reads `group`, ascending. -pub fn queries_rooted_at(queries: &[MemoQuery], group: MemoGroupId) -> Vec { - (0..queries.len()) - .filter(|&query| queries[query].root.operator_refs().contains(&&group)) - .collect() + /// The queries whose root reads `group`, ascending. + pub fn queries_rooted_at(&self, group: MemoGroupId) -> Vec { + (0..self.queries.len()) + .filter(|&query| self.queries[query].root.operator_refs().contains(&&group)) + .collect() + } } #[cfg(test)] diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 99a3d7ba9..5e436c7a9 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -8,20 +8,11 @@ use crate::ir::operator::AggIntent; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{ProjectItem, SortKey}; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; -use crate::ir::ASAPOp; use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; use crate::stages::logical::MemoGroupAsInput; use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; -use crate::test_support::memo_fixtures::{scan, state}; - -/// `SummaryEstimate[q](state)`. -fn quantile_readout(state: MemoGroupId, q: f64) -> Operator { - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: state, - query: SketchStatistic::Quantile { q }, - }) -} +use crate::test_support::memo_fixtures::{aggregate_where, readout, scan, state}; /// An unordered `LIMIT 10` over `input`, whose copies may return /// different rows. @@ -252,7 +243,9 @@ fn add_candidate_maintains_parents_and_queries() { let g2 = memo .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) .unwrap(); - let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); + let g3 = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.95 }, g2)) + .unwrap(); assert_eq!(memo.add_query(query(g3)), 0); // A rule adds a state group G5 over another scan G4: no query // reaches them yet. @@ -264,8 +257,11 @@ fn add_candidate_maintains_parents_and_queries() { .unwrap(); assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); // G3 gains a candidate reading G5: G5 and G4 now serve query 0. - memo.add_candidate(g3, candidate(quantile_readout(g5, 0.95), None)) - .unwrap(); + memo.add_candidate( + g3, + candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g5), None), + ) + .unwrap(); assert_eq!(memo.groups()[g5.0 as usize].parent_groups(), &[g3]); assert_eq!(memo.groups()[g4.0 as usize].parent_groups(), &[g5]); assert_eq!(memo.groups()[g5.0 as usize].reached_by_queries(), &[0]); @@ -297,7 +293,7 @@ fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { .unwrap(); let before = format!("{memo:?}"); assert!(matches!( - memo.add_candidate(g2, candidate(quantile_readout(g2, 0.5), None)), + memo.add_candidate(g2, candidate(readout(SketchStatistic::Quantile { q: 0.5 }, g2), None)), Err(CandidateError::OutputDiffersFromGroup { group }) if group == g2 )); assert_eq!( @@ -318,18 +314,26 @@ fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { let g2 = memo .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) .unwrap(); - let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); - let over_raw_rows = quantile_readout(g1, 0.95).map_children(|&group_id| MemoGroupAsInput { - memo: &memo, - group_id, - }); + let g3 = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.95 }, g2)) + .unwrap(); + let over_raw_rows = + readout(SketchStatistic::Quantile { q: 0.95 }, g1).map_children(|&group_id| { + MemoGroupAsInput { + memo: &memo, + group_id, + } + }); assert!( over_raw_rows.output_schema().is_ok(), "fixture: the schema derives; only input validation rejects it" ); let before = format!("{memo:?}"); assert!(matches!( - memo.add_candidate(g3, candidate(quantile_readout(g1, 0.95), None)), + memo.add_candidate( + g3, + candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g1), None) + ), Err(CandidateError::NotValidOverInputGroups(_)) )); assert_eq!( @@ -408,14 +412,12 @@ fn quantile_agg( filter: Option>, ) -> impl FnOnce(&mut Memo) -> MemoGroupId { over_scan(move |child| { - Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Quantile { col, q }], - output_names: vec![], - filters: vec![filter], - having: None, + aggregate_where( + Reduction::by(by), + AggIntent::Quantile { col, q }, + filter, child, - }) + ) }) } diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 2ca021553..234b6ddc7 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -19,6 +19,6 @@ pub use column_properties::ValueSign; pub use edge::LogicalEdge; use group::MemoGroupAsInput; pub use group::{GroupProperties, MemoGroup, MemoGroupId}; -pub use memo::{queries_rooted_at, root_groups, Memo, MemoQuery}; +pub use memo::{Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; pub use requirements::{LogicalRequirements, RequiredAccuracy}; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 180f2ad09..df594f8e5 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -13,11 +13,11 @@ mod physical; mod pruning; pub use logical::{ - queries_rooted_at, root_groups, AggregateCoverage, AggregateDefinition, CandidateError, - CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, CoverageError, - GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, - LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupId, MemoQuery, - OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, + AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, + CandidateForMemoGroup, CandidateForMemoGroupId, CoverageError, GroupProperties, + LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, + LogicalRequirements, Memo, MemoGroup, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, + RequiredAccuracy, ValueSign, }; pub use per_computation_logical_memo_candidates::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, diff --git a/crates/types/src/test_support/memo_fixtures.rs b/crates/types/src/test_support/memo_fixtures.rs index 351a8dbf1..6fc9b4d40 100644 --- a/crates/types/src/test_support/memo_fixtures.rs +++ b/crates/types/src/test_support/memo_fixtures.rs @@ -4,11 +4,12 @@ use std::time::Duration; use crate::ir::operator::arguments::{AtModifier, Reduction, Source, TimeShift}; +use crate::ir::operator::AggIntent; use crate::ir::scalar::ColumnRef; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; use crate::ir::{ASAPOp, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; use crate::stages::MemoGroupId; -use crate::summary_type::SummaryUpdate; +use crate::summary_type::{SketchStatistic, SummaryUpdate}; /// `Scan metric{predicates}`: `[ts, value, columns…]`, `ts` its time index. pub fn scan( @@ -70,3 +71,38 @@ pub fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator Operator { + aggregate_where(reduction, measure, None, child) +} + +/// `Aggregate(reduction; measure)` over `child`, the measure over the rows +/// `filter` keeps (`None`: every row). +pub fn aggregate_where( + reduction: Reduction, + measure: AggIntent, + filter: Option>, + child: MemoGroupId, +) -> Operator { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures: vec![measure], + output_names: vec![], + filters: filter.map_or_else(Vec::new, |filter| vec![Some(filter)]), + having: None, + child, + }) +} + +/// `SummaryEstimate[query](state)`: a readout of `state`. +pub fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query, + }) +} diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 8eaec83de..005db69a0 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -24,7 +24,9 @@ use asap_types::ir::{ }; use asap_types::stages::{Memo, MemoGroupId, ValueSign}; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; -use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; +use asap_types::test_support::memo_fixtures::{ + aggregate, filter, range, readout, scan, shift, state, +}; use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; use ValueSign::{NonNegative, NonPositive, Unknown}; @@ -109,32 +111,10 @@ fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator Operator { - Operator::NonASAP(NonASAPOp::Aggregate { - reduction, - measures: vec![measure], - output_names: vec![], - filters: vec![], - having: None, - child, - }) -} - fn by_host() -> Reduction { Reduction::by(vec![HOST]) } -fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator { - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: state, - query, - }) -} - /// Rows of the per-operator table that keep or set a column's sign: /// /// | Group | Expression | `column_signs` | diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index e80626db9..2201ecb76 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -960,7 +960,7 @@ MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 | §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.summary_type` | | §6.4 Summary algorithms and parameters | `SketchAlgorithm`, `SketchParams`, `HydraParams`, exact/sample/wavelet/model kinds and params | the payload of each non-plain `FieldDataType` variant; θ on a candidate's `LogicalProperties` | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | -| §6.6 Summary coverage | `AggregateCoverage`, `AggregateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of summary-state memo groups | +| §6.6 Aggregate coverage | `AggregateCoverage`, `AggregateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of aggregate memo groups | **Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `AggregateCoverage` (§6.6). diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 2468e8daa..d4e04bcbe 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -17,7 +17,7 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | Shared vocabulary, by scope: the operator IR by IR section, `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator arguments, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`), `ir::properties` (execution timing, summary coverage) and `ir::flat`; summary types (`summary_type`: `SketchAlgorithm`, `SketchParams`, `SummaryTypeCapability`); accuracy targets and guarantees (`accuracy`: `AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`); the planner inputs (`workload`); the stage-boundary types (`stages`, with Stage 2's `PhysicalProperties` (timing and `Lifetime`)); and Stage 3's plan (`plan`: `plan::physical_asap_dag`, `plan::materialized_view`) | +| `asap-types` (`crates/types`) | Shared vocabulary, by scope: the operator IR by IR section, `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator arguments, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`), `ir::properties` (execution timing, the selection algebra of aggregate coverage) and `ir::flat`; summary types (`summary_type`: `SketchAlgorithm`, `SketchParams`, `SummaryTypeCapability`); accuracy targets and guarantees (`accuracy`: `AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`); the planner inputs (`workload`); the stage-boundary types (`stages`, with Stage 2's `PhysicalProperties` (timing and `Lifetime`)); and Stage 3's plan (`plan`: `plan::physical_asap_dag`, `plan::materialized_view`) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | | `asap-stage0-memo` (`crates/stage0-memo`) | Stage 0 hash-consing: the memo groups of the workload's query roots (`memo_of_queries`) | | `asap-stage1-pass1-per-computation-candidates` (`crates/stage1-pass1-per-computation-candidates`) | Stage 1, Pass 1: the initial candidates of each memo group | From bba5e221cf38cea7c7f48ab885fe79f8c4e18a4f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:03:04 +0000 Subject: [PATCH 12/12] Aggregate coverage in the IR, generic over the input reference; the plan's content id by expansion - AggregateCoverage, AggregateDefinition (input_group renamed input) and CoverageError move to ir/properties/aggregate_coverage.rs with the derivation, now Operator::::aggregate_coverage, generic over OperatorInput::Reference (how a definition names an input). OperatorInput gains `type Reference` and `reference()`; node trees use `()` and store no coverage; the memo instantiates R = MemoGroupId. AggregateCoverage::map_references maps the references. Nothing under ir/ imports crate::stages. - ViewCoverage moves beside PhysicalASAPDAGNode. stages::view_coverage expands a group's coverage (every input group to its expression, down to the Scans) and hashes its JSON (FNV-1a 128) into definition_id; the selection is the memo's. Stage 3's todo names it. - Tests: equal definition_id across memos with different group ids, another source gives another id, no coverage gives None. - asap-primitive-schema.md: the IR type and derivation, map_references, the content id by expansion; derive_from_inputs and input_group gone. --- crates/planner/src/lib.rs | 4 +- crates/stage3-plan-selection/src/lib.rs | 5 +- crates/types/src/ir/operator/input.rs | 37 +- .../src/ir/properties/aggregate_coverage.rs | 781 +++++++++++++++++ crates/types/src/ir/properties/mod.rs | 12 +- crates/types/src/ir/properties/selection.rs | 2 +- crates/types/src/ir/schema/error.rs | 2 +- crates/types/src/plan/materialized_view.rs | 37 +- crates/types/src/plan/physical_asap_dag.rs | 38 +- .../src/stages/logical/aggregate_coverage.rs | 808 ++---------------- crates/types/src/stages/logical/group.rs | 18 +- crates/types/src/stages/logical/mod.rs | 2 +- crates/types/src/stages/mod.rs | 9 +- crates/types/tests/memo_aggregate_coverage.rs | 82 +- .../physical-planning-and-deployment.md | 4 +- .../proposals/asap-primitive-schema.md | 86 +- 16 files changed, 1058 insertions(+), 869 deletions(-) create mode 100644 crates/types/src/ir/properties/aggregate_coverage.rs diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index c88e260f7..b58aa2ec4 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -63,9 +63,9 @@ pub use asap_types::ir::properties::selection::{ pub use asap_types::ir::properties::ExecutionTiming; pub use asap_types::plan::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, MaterializedViewInputs, - MaterializedViewRefresh, RawDataRetention, RawDataWindow, RefreshWindow, ViewCoverage, + MaterializedViewRefresh, RawDataRetention, RawDataWindow, RefreshWindow, }; -pub use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +pub use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId, ViewCoverage}; pub use asap_types::summary_type::{SketchAlgorithm, SketchParams, StatisticKind}; pub use asap_types::workload::PlanningWorkload; diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 607d9386d..6452f0bed 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -29,7 +29,8 @@ pub use report::{ /// (module documentation) under `cost_inputs`, and build it: one /// `PhysicalASAPDAG` node per selected execution, its inputs from the /// selected edges, each node's `selected_physical_candidate` set and each -/// materialized view node's `view_coverage` from its memo group. +/// materialized view node's `view_coverage` from its memo group +/// (`asap_types::stages::view_coverage`). /// `entries[q]` is query `q`'s workload entry; `data` the data workload /// with its bounds. /// @@ -45,5 +46,5 @@ pub fn select_plan( .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; let _ = (physical, entries, data); - todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage from its memo group's aggregate_coverage, with definition_id the content id of the expanded definition") + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage = asap_types::stages::view_coverage(memo, its group)") } diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index 4c53e4c37..3cc9a088e 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -10,9 +10,9 @@ use std::rc::Rc; use super::arguments::Source; use super::node::{Operator, OperatorNode, OperatorResultKind}; use super::non_asap::NonASAPOp; +use crate::ir::properties::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; -use crate::stages::AggregateCoverage; /// An input of an operator, as the rules read it. pub trait OperatorInput: Sized + Clone { @@ -20,16 +20,22 @@ pub trait OperatorInput: Sized + Clone { fn result_kind(&self) -> OperatorResultKind; /// The input's own operator, its children as inputs again. fn operator(&self) -> Operator; - /// What the input covers when it is an aggregate group in the memo; - /// `None` for any other input. - fn aggregate_coverage(&self) -> Option<&AggregateCoverage>; + /// How an aggregate's definition names this input + /// ([`AggregateCoverage`]): a memo group's id in the memo. + type Reference: Clone + PartialEq; + /// This input's reference; `None` for a build a rule constructs only + /// while it derives, which has none. + fn reference(&self) -> Option; + /// What the input covers when its coverage is stored (an aggregate + /// group in the memo); `None` for any other input. + fn aggregate_coverage(&self) -> Option<&AggregateCoverage>; /// The sources the `Scan`s at or below the input read /// (`Operator::scanned_sources` of its operator): a memo group's stored /// `GroupProperties::scanned_sources`, recomputed for a node tree. fn scanned_sources(&self) -> BTreeSet; } -/// A front-end node tree. Node trees carry no aggregate coverage. +/// A front-end node tree. Node trees store no aggregate coverage. impl OperatorInput for Rc { fn schema(&self) -> &Schema { &self.schema @@ -43,7 +49,15 @@ impl OperatorInput for Rc { self.operator.clone() } - fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { + /// A node tree is named by nothing: no definition over a node tree is + /// stored or compared. + type Reference = (); + + fn reference(&self) -> Option<()> { + Some(()) + } + + fn aggregate_coverage(&self) -> Option<&AggregateCoverage<()>> { None } @@ -119,7 +133,16 @@ impl OperatorInput for InputOrTemporaryBuild { } } - fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { + type Reference = I::Reference; + + fn reference(&self) -> Option { + match self { + Self::Input(input) => input.reference(), + Self::TemporaryBuild(_) => None, + } + } + + fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { match self { Self::Input(input) => input.aggregate_coverage(), Self::TemporaryBuild(_) => None, diff --git a/crates/types/src/ir/properties/aggregate_coverage.rs b/crates/types/src/ir/properties/aggregate_coverage.rs new file mode 100644 index 000000000..fbfca1122 --- /dev/null +++ b/crates/types/src/ir/properties/aggregate_coverage.rs @@ -0,0 +1,781 @@ +//! Aggregate coverage, an IR property derived like an output schema: what +//! an aggregate (a summary state or an exact `Aggregate`) covers +//! ([`AggregateCoverage`]): `definition` (what it computes) and `selection` +//! (which output rows of that computation it took, in the selection algebra +//! of [`super::selection`]). Design: +//! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after +//! Goldstein & Larson's view matching. +//! +//! It is derived over an operator's inputs ([`Operator::aggregate_coverage`]), +//! generic over how a definition names an input +//! ([`OperatorInput::Reference`]): the memo derives it once per aggregate +//! group (`R` = `MemoGroupId`) and stores it in +//! `GroupProperties::aggregate_coverage`; a node tree stores none. Walking +//! down from a `SummaryAgg` or `Aggregate` through its inputs' operators, a +//! predicate conjunct moves into `selection` when it can be lifted to the +//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and +//! direct-column `Project` items) and it is a value set or an interval on +//! one column. The `TimeRange` and `TimeShift`s become one interval on the +//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! `definition` as a residual, so two states are merged only when they +//! compute the same thing over disjoint rows. + +use std::collections::BTreeMap; +use std::ops::Bound; + +use thiserror::Error; + +use super::selection::{ + constraint_of, union, ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, +}; +use crate::ir::operator::arguments::{AtModifier, Reduction}; +use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; +use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; +use crate::ir::operator::node::{Operator, OperatorResultKind}; +use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; +use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::scalar::{Predicate, ScalarExpr}; +use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::SchemaDerivationError; +use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; + +/// What an aggregate computes, with every lifted selection removed. +/// `R` is how it names an input: a memo group's id in the memo. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct AggregateDefinition { + /// The input the unlifted operators read: the first input under the + /// `SummaryAgg` or exact `Aggregate` that is not a `Filter`, a range + /// `TimeRange`, a `TimeShift` without `@ start()`/`@ end()` or a + /// direct-column `Project`. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator, + /// so states over one source with different matchers may merge. + pub input: Option, + /// The operators not lifted into the selection, listed top down from the + /// `SummaryAgg` or `Aggregate`, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or `input`); + /// `Some(r)` is another input it reads, such as the input a scalar + /// subquery in a Filter reads. Two states may merge only if `input` + /// and these operators are equal. + pub unlifted_operators: Vec>>, +} + +/// What an aggregate covers, its inputs named by `R`. +#[derive(Debug, Clone, PartialEq)] +pub struct AggregateCoverage { + pub(crate) definition: AggregateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. + pub(crate) selection: Vec, +} + +impl AggregateCoverage { + /// The same coverage with each input reference `r` replaced by `f(r)`. + pub fn map_references(self, mut f: impl FnMut(R) -> S) -> AggregateCoverage { + let AggregateDefinition { + input, + unlifted_operators, + } = self.definition; + AggregateCoverage { + definition: AggregateDefinition { + input: input.map(&mut f), + unlifted_operators: unlifted_operators + .into_iter() + .map(|operator| operator.map_children(|child| child.clone().map(&mut f))) + .collect(), + }, + selection: self.selection, + } + } +} + +#[cfg(any(test, feature = "test-support"))] +impl AggregateCoverage { + pub fn definition(&self) -> &AggregateDefinition { + &self.definition + } + + /// Union of boxes over the output rows of the unlifted operators' input. + pub fn selection(&self) -> &[SelectionBox] { + &self.selection + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CoverageError { + /// The node has no coverage: it is no aggregate (`SummaryAgg`, + /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a + /// `TimeRange`/`TimeShift` over a chain whose windows are not one + /// interval on the timestamp column. A caller bug, or bad input when it + /// is a merge's input. + #[error("coverage is derived only for aggregates and one time interval over a chain")] + NoCoverage, + /// A `SummaryMerge` without inputs: bad input. + #[error("summary merge requires at least one input")] + EmptyMerge, + /// The inputs of a `SummaryMerge` compute different things: bad input. + #[error("summary merge inputs compute different things")] + DefinitionMismatch, + /// The inputs of a `SummaryMerge` are not proven to relate as the + /// family's merge requires (disjoint, or disjoint in time for counter + /// states): bad input. + #[error( + "summary merge inputs are not proven {}, which the family's merge requires", + match required { + MergeInputRule::DisjointInTime => "disjoint in time", + _ => "disjoint", + } + )] + InputsOverlap { required: MergeInputRule }, + /// A regrouping `SummaryMerge` key that is not one of the group keys + /// its input states are built by (a build grouped `without` keys has + /// none): bad input. + #[error("regrouping merge key {key} is not a group key of its input states")] + NotAGroupKey { key: ColumnId }, + /// A regrouping `SummaryMerge` of states whose summary type does not + /// merge different groups: counter states (`Rate`, `Increase`) merge + /// consecutive windows of one series, a summary type without a merge + /// merges nothing, and a Hydra's groups share one state. Bad input. + #[error("the inputs' summary type cannot merge different groups")] + SummaryTypeCannotRegroup, + /// The build a regrouping `SummaryMerge` stands for does not derive: + /// bad input. + #[error("the regrouped build does not derive: {reason}")] + RegroupedBuild { reason: String }, +} + +impl Operator { + /// The coverage of what this operator computes over its inputs + /// (`coverage_of`, below). + pub fn aggregate_coverage(&self) -> Result, CoverageError> { + coverage_of(self) + } +} + +/// Coverage of what `operator` computes over its inputs: a `SummaryAgg`, an exact `Aggregate`, a +/// `SummaryTumblingWindowChain` (a build over its input, the timestamp +/// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the +/// chain's coverage with the windows' interval on the timestamp column), +/// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped +/// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` +/// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge +/// reads its inputs' stored coverage and fails +/// unless every input has the same definition and their selections are +/// pairwise disjoint; a family whose merge is idempotent +/// ([`MergeInputRule::MayOverlap`] in the family-capability table) may +/// also merge overlapping selections, and a counter family +/// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. +fn coverage_of( + operator: &Operator, +) -> Result, CoverageError> { + let as_built = |op: &ASAPOp| -> ASAPOp> { + op.map_children(|input| InputOrTemporaryBuild::Input(input.clone())) + }; + match operator { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = unlifted( + &operator.map_children(|input| InputOrTemporaryBuild::Input(input.clone())), + ); + of_aggregate(&InputOrTemporaryBuild::Input(child.clone()), None, |_| top) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if operator.output_kind() == OperatorResultKind::State => + { + of_window_selection(operator) + } + Operator::NonASAP(_) => match selected_build(operator)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NoCoverage), + }, + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => of_summary_agg(&as_built(op)), + Operator::ASAP(ASAPOp::SummaryMerge { + children, group_by, .. + }) => { + let inputs = children + .iter() + .map(|child| { + child + .aggregate_coverage() + .cloned() + .ok_or(CoverageError::NoCoverage) + }) + .collect::, _>>()?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; + match group_by { + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), + } + } + Operator::ASAP(_) => Err(CoverageError::NoCoverage), + } +} + +/// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by +/// their key columns, the build that computes exactly the kept states: the +/// same build over its input rows filtered by the same predicate on the +/// input columns those keys name. `None` for any other operator. +fn selected_build( + operator: &Operator, +) -> Result>>, CoverageError> { + let Operator::NonASAP(NonASAPOp::Filter { + pred, + child: states, + }) = operator + else { + return Ok(None); + }; + if operator.output_kind() != OperatorResultKind::State { + return Ok(None); + } + let Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: family, + input, + reduction: Reduction::Reduce(by), + filter, + }) = states.operator() + else { + return Err(CoverageError::NoCoverage); + }; + if by.is_without() { + return Err(CoverageError::NoCoverage); + } + // Each state key column is the input column of the same name it groups by. + let mut unmapped = None; + let rows_pred = pred.0.map_columns(&mut |column| { + let mapped = by.keys().get(column).copied().filter(|&input_column| { + matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), + (Some(at_state), Some(at_input)) if at_state.name == at_input.name) + }); + mapped.unwrap_or_else(|| { + unmapped = Some(column); + column + }) + }); + if let Some(key) = unmapped { + return Err(CoverageError::NotAGroupKey { key }); + } + let mut built = |input: &I| InputOrTemporaryBuild::Input(input.clone()); + let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { + reason: error.to_string(), + }; + let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { + pred: Predicate(rows_pred.map_operator_refs(&mut built)), + child: InputOrTemporaryBuild::Input(child), + }) + .map_err(regrouped_build)?; + let reduction = Reduction::Reduce(by); + summary_agg_layout(rows.schema(), &family, &reduction).map_err(regrouped_build)?; + Ok(Some(ASAPOp::SummaryAgg { + child: rows, + summary_type: family, + input, + reduction, + filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), + })) +} + +/// The coverage of a merge of states covering `inputs`, whose builds read +/// rows with timestamp column `time_column`. +fn of_merge( + inputs: &[AggregateCoverage], + time_column: Option<&ColumnIdentity>, +) -> Result, CoverageError> { + let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; + if inputs + .iter() + .any(|input| input.definition != first.definition) + { + return Err(CoverageError::DefinitionMismatch); + } + // A family without a merge is checked as the strictest rule; the + // executor rejects the merge itself. + let required = match first.definition.unlifted_operators.first() { + Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + })) => SummaryTypeCapability::of(family).merge_inputs, + _ => None, + } + .unwrap_or(MergeInputRule::Disjoint); + let separated = |a: &SelectionBox, b: &SelectionBox| match required { + MergeInputRule::MayOverlap => true, + // Consecutive windows of one series: disjoint on the timestamp column. + MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { + matches!( + (a.columns.get(column), b.columns.get(column)), + (Some(a), Some(b)) if a.disjoint(b) + ) + }), + MergeInputRule::Disjoint => a.disjoint(b), + }; + // Every pair of boxes of different inputs. + if required != MergeInputRule::MayOverlap + && inputs.iter().enumerate().any(|(i, input)| { + inputs[..i].iter().any(|other| { + input + .selection + .iter() + .any(|a| other.selection.iter().any(|b| !separated(a, b))) + }) + }) + { + return Err(CoverageError::InputsOverlap { required }); + } + Ok(AggregateCoverage { + definition: first.definition.clone(), + selection: union(inputs.iter().flat_map(|c| c.selection.clone()).collect()), + }) +} + +/// The coverage of a regrouping merge, keyed by `keys`, of states whose +/// merge covers `merged` (`first` is its first input): the same rows +/// (`selection`), computed by the input's build grouped by the columns +/// `keys` name. The input states of one output group are different groups +/// of one build, so their rows are disjoint. A counter family merges only +/// time-disjoint windows, never different groups at one time. +fn regrouped( + merged: AggregateCoverage, + first: &I, + keys: &[ColumnId], +) -> Result, CoverageError> { + let Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + input, + reduction, + filter, + .. + })) = merged.definition.unlifted_operators.first() + else { + return Err(CoverageError::NoCoverage); + }; + if !SummaryTypeCapability::merges_groups(family) { + return Err(CoverageError::SummaryTypeCannotRegroup); + } + let state = first.schema(); + let rows = build_input_schema(first).ok_or(CoverageError::NoCoverage)?; + // The build's input column each key of the state names. + let mapped = keys + .iter() + .map(|&key| { + let column = match reduction { + Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), + // A per-series state keeps its input's columns in place. + Reduction::PerEntity => Some(key), + Reduction::Reduce(_) => None, + }; + column + .filter(|&column| { + let (Some(at_state), Some(at_input)) = + (state.fields.get(key), rows.fields.get(column)) + else { + return false; + }; + at_state.is_plain() + && at_state.name == at_input.name + && at_state.table == at_input.table + }) + .ok_or(CoverageError::NotAGroupKey { key }) + }) + .collect::, _>>()?; + let reduction = Reduction::by(mapped); + summary_agg_layout(&rows, family, &reduction).map_err(|error| { + CoverageError::RegroupedBuild { + reason: error.to_string(), + } + })?; + // A direct build's definition, so it is `==` to one. + let mut definition = merged.definition.clone(); + definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), + input: input.clone(), + reduction, + filter: filter.clone(), + }); + Ok(AggregateCoverage { + definition, + selection: merged.selection, + }) +} + +/// The schema of the rows the build of summary state `state` reads (its +/// `SummaryAgg`'s input; one build over a chain's window reads its range of +/// the chain's input, which has the input's schema). `None` when `state` +/// is no summary state. +fn build_input_schema(state: &I) -> Option { + if let Some(states) = selected_from(state) { + return build_input_schema(&states); + } + match state.operator() { + Operator::ASAP( + ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, + ) => Some(child.schema().clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { + build_input_schema(children.first()?) + } + _ => None, + } +} + +/// The timestamp column of the rows the build of summary state `state` +/// reads ([`build_input_schema`]), as a selection names it. +fn time_column_of_build_input(state: &I) -> Option { + let rows = build_input_schema(state)?; + column_identity(&rows, rows.time_index?) +} + +/// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` +/// over inputs (or over an input a rule built over them): a chain covers +/// what its `SummaryAgg` over the same input would. +fn of_summary_agg( + build: &ASAPOp>, +) -> Result, CoverageError> { + let (ASAPOp::SummaryAgg { + child, + summary_type: family, + input, + reduction, + filter, + } + | ASAPOp::SummaryTumblingWindowChain { + child, + summary_type: family, + input, + reduction, + filter, + .. + }) = build + else { + unreachable!("called on a SummaryAgg or a chain"); + }; + of_aggregate(child, filter.as_ref(), |residual_filter| { + Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + filter: residual_filter, + }) + }) +} + +/// The coverage of an aggregate over `child`: its input's operators walked +/// down through the liftable operators of its inputs' operators. +/// `filter` is the aggregate's own row filter (a `SummaryAgg`'s), and +/// `unlifted_top` makes the aggregate's unlifted operator from that filter's residual. +fn of_aggregate( + child: &InputOrTemporaryBuild, + filter: Option<&Predicate>>, + unlifted_top: impl FnOnce(Option>>) -> Operator>, +) -> Result, CoverageError> { + // The operators a predicate can be lifted through, top to bottom, and + // the first input below them. A subquery's input stays named in the + // residual of the operator that reads it. + let mut chain: Vec>> = Vec::new(); + let mut base = child.clone(); + loop { + let operator = base.operator(); + let passes = match &operator { + Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, + // `@ end()` / `@ start()` pin their subtree to one end of the + // request: not lifted, so their group is the input group. + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + !matches!(shift.at, Some(AtModifier::Start | AtModifier::End)) + } + _ => false, + }; + if !passes { + break; + } + // The main input is an operator's first child. + base = operator.children()[0].clone(); + chain.push(operator); + } + let InputOrTemporaryBuild::Input(base_input) = &base else { + unreachable!("a rule builds only liftable operators over real inputs"); + }; + let top = child.schema(); + // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual + // of `chain[d]` when it is a `Filter`. + let mut lifted = BTreeMap::new(); + let agg_filter = filter.and_then(|pred| residual(pred, top, &[], &mut lifted)); + let kept: Vec>>> = chain + .iter() + .enumerate() + .map(|(depth, link)| match link { + Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { + residual(pred, top, &chain[..depth], &mut lifted) + } + _ => None, + }) + .collect(); + // A `Scan`'s predicates (PromQL label matchers) are lifted too; when + // one is, the `Scan` with the rest of its predicates is the last + // unlifted operator, and there is no input. + let scan = match base.operator() { + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) => { + let lifted_above = lifted.clone(); + let kept = predicates + .iter() + .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) + .collect(); + (lifted != lifted_above).then_some(NonASAPOp::Scan { + source, + predicates: kept, + schema, + }) + } + _ => None, + }; + + // The `TimeRange` and `TimeShift`s: one interval on the timestamp column + // of the `TimeRange`'s input, when they make one and it combines with + // what is lifted on that column already; else they stay unlifted. + let time = time_constraint(&chain).and_then(|constraint| { + let depth = chain + .iter() + .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; + let column = chain[depth].children()[0].schema().time_index?; + let column = column_at_top(column, &chain[..=depth])?; + column_identity(top, column)?; + Some((column, combined_with(lifted.get(&column), constraint)?)) + }); + let lift_time = time.is_some(); + lifted.extend(time); + + let mut unlifted_operators = vec![unlifted_top(agg_filter)]; + for (depth, link) in chain.iter().enumerate() { + match link { + Operator::NonASAP(NonASAPOp::Filter { .. }) => { + if let Some(rest) = kept[depth].clone() { + unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { + pred: rest, + child: None, + })); + } + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if lift_time => {} + other => unlifted_operators.push(unlifted(other)), + } + } + let input = match scan { + Some(scan) => { + unlifted_operators.push(Operator::NonASAP(scan)); + None + } + None => base_input.reference(), + }; + + let selection = SelectionBox { + columns: lifted + .into_iter() + .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) + .collect(), + }; + Ok(AggregateCoverage { + definition: AggregateDefinition { + input, + unlifted_operators, + }, + selection: vec![selection], + }) +} + +/// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of +/// `top`, the agg child's schema) and return the rest, each scalar subquery +/// naming the input it reads. `above` +/// are the operators between the predicate and the `SummaryAgg`, top to +/// bottom. A column [`column_identity`] cannot name stays. +fn residual( + pred: &Predicate>, + top: &Schema, + above: &[Operator>], + lifted: &mut BTreeMap, +) -> Option>> { + let mut rest: Vec>> = Vec::new(); + for conjunct in pred.0.conjuncts() { + let lift = constraint_of(conjunct).and_then(|(column, constraint)| { + let column = column_at_top(column, above)?; + column_identity(top, column)?; + Some((column, combined_with(lifted.get(&column), constraint)?)) + }); + match lift { + Some((column, constraint)) => { + lifted.insert(column, constraint); + } + None => rest.push(conjunct.map_operator_refs(&mut reference_of)), + } + } + match rest.len() { + 0 => None, + 1 => rest.pop().map(Predicate), + _ => Some(Predicate(ScalarExpr::BoolAnd(rest))), + } +} + +/// `constraint` combined with what is lifted on its column already +/// (`existing`); `None` when the two do not combine. +fn combined_with(existing: Option<&Constraint>, constraint: Constraint) -> Option { + match existing { + Some(existing) => existing.intersect(&constraint), + None => Some(constraint), + } +} + +/// How a definition names `input`; `None` for a rule's temporary build. +fn reference_of(input: &InputOrTemporaryBuild) -> Option { + input.reference() +} + +/// `link`, a liftable operator or an exact `Aggregate` kept whole in a +/// definition: its main input `None`, every other input its reference. +fn unlifted( + link: &Operator>, +) -> Operator> { + let mut operator = link.map_children(reference_of); + match &mut operator { + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::Project { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. } + | NonASAPOp::Aggregate { child, .. }, + ) => *child = None, + _ => unreachable!("only liftable operators and aggregates are unlifted"), + } + operator +} + +/// Map a column of the input of `above.last()` up to the agg child's output: +/// a `Project` passes it only as a direct column item. +fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { + for link in above.iter().rev() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { + column = cols + .iter() + .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; + } + } + Some(column) +} + +/// The identity of `schema`'s column `column`; `None` when its +/// `(table, name)` is not unique in `schema`, so a selection cannot name it. +fn column_identity(schema: &Schema, column: ColumnId) -> Option { + let field = schema.fields.get(column)?; + let namesakes = schema + .fields + .iter() + .filter(|f| f.name == field.name && f.table == field.table) + .count(); + (namesakes == 1).then(|| ColumnIdentity { + table: field.table.clone(), + name: field.name.clone(), + }) +} + +/// The interval on the timestamp column that `links` (top down, the +/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: +/// `(anchor − o − W, anchor − o]` for the one range `TimeRange` of width W +/// and the shifts' offsets summing to o, left-open and right-closed like a +/// PromQL range. The anchor is the evaluation time t +/// (`EvaluationTimePlusMs`), or T for `@ T` (`FixedValue(Timestamp)`). +/// `None` when they are not one interval: no or several range `TimeRange`s, +/// an instant one, several `@`s, or an `@ end()` or `@ start()`, which pins +/// its subtree to one end of the request and is never lifted (the front +/// ends put it above the subtree it anchors). +fn time_constraint(links: &[Operator]) -> Option { + let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); + for link in links { + match link { + Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { + if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { + return None; + } + } + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + offset_ms = offset_ms.checked_add(shift.offset_ms)?; + if let Some(anchor) = shift.at { + if at.replace(anchor).is_some() { + return None; + } + } + } + _ => {} + } + } + let width_ms = i64::try_from(width?.as_millis()).ok()?; + let fixed_anchor = match at { + Some(AtModifier::Timestamp(ms)) => Some(ms), + Some(AtModifier::Start | AtModifier::End) => return None, + None => None, + }; + let before_anchor = |ms: i64| -> Option { + Some(match fixed_anchor { + Some(anchor) => { + IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) + } + None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), + }) + }; + Some(Constraint::Interval { + lower: Bound::Excluded(before_anchor(offset_ms.checked_add(width_ms)?)?), + upper: Bound::Included(before_anchor(offset_ms)?), + }) +} + +/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain +/// (through more of them): the chain's coverage with the interval they +/// select on the timestamp column of the chain's input, in every box. +fn of_window_selection( + selection: &Operator, +) -> Result, CoverageError> { + let mut links = vec![selection.clone()]; + let mut below = selection.children()[0].clone(); + while let Some(child) = selected_from(&below) { + links.push(below.operator()); + below = child; + } + let only_time_selections = links.iter().all(|link| { + matches!( + link, + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + ) + }); + if !only_time_selections + || !matches!( + below.operator(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + { + return Err(CoverageError::NoCoverage); + } + let chain = below + .aggregate_coverage() + .ok_or(CoverageError::NoCoverage)?; + let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NoCoverage)?; + let constraint = time_constraint(&links).ok_or(CoverageError::NoCoverage)?; + let selection = chain + .selection + .iter() + .map(|selection_box| { + let mut columns = selection_box.columns.clone(); + let combined = combined_with(columns.get(&time_column), constraint.clone()) + .ok_or(CoverageError::NoCoverage)?; + columns.insert(time_column.clone(), combined); + Ok(SelectionBox { columns }) + }) + .collect::>()?; + Ok(AggregateCoverage { + definition: chain.definition.clone(), + selection, + }) +} diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 380b0bd8b..dff3b6bd9 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -1,14 +1,18 @@ //! #511 §2.2–2.3: the planning properties a node carries besides its -//! operator and schema (when its value is produced), and the selection -//! algebra an aggregate's coverage (`stages::AggregateCoverage`) is stated -//! in. +//! operator and schema (when its value is produced), and what an aggregate +//! covers with the selection algebra it is stated in. //! //! - [`execution_data_state`] — when a node's value is produced //! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). +//! - [`aggregate_coverage`] — what an aggregate computes and which rows it +//! took ([`AggregateCoverage`]), derived over an operator's inputs. //! - [`selection`] — which rows of a computation an aggregate's state took: //! boxes of per-column constraints, the selection algebra aggregate -//! coverage (`stages::AggregateCoverage`) is built from. +//! coverage is built from. +pub mod aggregate_coverage; pub mod execution_data_state; pub mod selection; +pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; + pub use execution_data_state::{DataPrimitive, ExecutionDataState, ExecutionTiming}; diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs index 7b9351594..6bd47b5de 100644 --- a/crates/types/src/ir/properties/selection.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -1,5 +1,5 @@ //! The selection algebra of aggregate coverage -//! (`stages::AggregateCoverage::selection`): which output rows of a +//! ([`AggregateCoverage`](super::AggregateCoverage)'s `selection`): which output rows of a //! computation a state took, as a union of boxes ([`SelectionBox`]), each a //! [`Constraint`] per column ([`ColumnIdentity`]): value sets and intervals, //! with interval limits relative to the evaluation time diff --git a/crates/types/src/ir/schema/error.rs b/crates/types/src/ir/schema/error.rs index f10cf38d5..e84a61b43 100644 --- a/crates/types/src/ir/schema/error.rs +++ b/crates/types/src/ir/schema/error.rs @@ -17,5 +17,5 @@ pub enum SchemaDerivationError { #[error("invalid per-series sample column: {0}")] InvalidSampleColumn(String), #[error("invalid aggregate coverage: {0}")] - Coverage(#[from] crate::stages::CoverageError), + Coverage(#[from] crate::ir::properties::CoverageError), } diff --git a/crates/types/src/plan/materialized_view.rs b/crates/types/src/plan/materialized_view.rs index 2a87062fb..5915bf633 100644 --- a/crates/types/src/plan/materialized_view.rs +++ b/crates/types/src/plan/materialized_view.rs @@ -70,13 +70,13 @@ use thiserror::Error; use super::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, + ViewCoverage, }; use crate::ir::operator::arguments::Source; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; -use crate::ir::properties::selection::SelectionBox; use crate::stages::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is @@ -118,41 +118,6 @@ impl MaterializedView { } } -/// A view's aggregate coverage -/// ([`AggregateCoverage`](crate::stages::AggregateCoverage), `docs/design_docs/proposals/asap-primitive-schema.md` -/// §4.2): what it computes and which rows went in. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ViewCoverage { - /// The plan-scope content id of what the view computes: its memo - /// coverage's `AggregateDefinition` with every input group expanded - /// recursively to its expression, down to the `Scan`s, holding no - /// memo group or plan node ids, canonically serialized and hashed. - /// Equal across plans and processes that compute the same thing, - /// whatever the selection, and different when they do not (a rollup to - /// a coarser grouping computes something else and has its own). Inside - /// one memo, coverages compare by `AggregateDefinition` `==` instead - /// and need no id. - pub(crate) definition_id: String, - /// The rows that went in: a union of boxes of value sets, value ranges - /// and time relative to the evaluation time. A merge's is the union of - /// its inputs'. - pub(crate) selection: Vec, -} - -#[cfg(any(test, feature = "test-support"))] -impl ViewCoverage { - /// The plan-scope content id of what the view computes. - pub fn definition_id(&self) -> &str { - &self.definition_id - } - - /// The rows that went in. - pub fn selection(&self) -> &[SelectionBox] { - &self.selection - } -} - /// What a view's refresh reads, each bound as a source of the refresh. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index 9d108ade6..730e144b5 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -15,7 +15,6 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use super::materialized_view::ViewCoverage; use crate::ir::flat::NodeId; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::Operator; @@ -26,6 +25,7 @@ use crate::ir::scalar::ScalarExpr; use crate::accuracy::AccuracyGuarantee; use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; +use crate::ir::properties::selection::SelectionBox; use crate::ir::schema::Schema; use crate::stages::PhysicalCandidateId; use crate::stages::{Lifetime, PhysicalPropertiesError}; @@ -109,6 +109,42 @@ pub struct PhysicalASAPDAGNode { pub view_coverage: Option, } +/// A view's aggregate coverage +/// ([`AggregateCoverage`](crate::ir::properties::AggregateCoverage), `docs/design_docs/proposals/asap-primitive-schema.md` +/// §4.2): what it computes and which rows went in. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ViewCoverage { + /// The plan-scope content id of what the view computes: its memo + /// coverage's `AggregateDefinition` with every input group expanded + /// recursively to its expression, down to the `Scan`s, holding no + /// memo group or plan node ids, canonically serialized and hashed + /// (`stages::view_coverage`). + /// Equal across plans and processes that compute the same thing, + /// whatever the selection, and different when they do not (a rollup to + /// a coarser grouping computes something else and has its own). Inside + /// one memo, coverages compare by `AggregateDefinition` `==` instead + /// and need no id. + pub(crate) definition_id: String, + /// The rows that went in: a union of boxes of value sets, value ranges + /// and time relative to the evaluation time. A merge's is the union of + /// its inputs'. + pub(crate) selection: Vec, +} + +#[cfg(any(test, feature = "test-support"))] +impl ViewCoverage { + /// The plan-scope content id of what the view computes. + pub fn definition_id(&self) -> &str { + &self.definition_id + } + + /// The rows that went in. + pub fn selection(&self) -> &[SelectionBox] { + &self.selection + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PhysicalASAPDAGEdge { diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index b06224c12..a8a80d550 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -1,753 +1,59 @@ -//! Aggregate coverage, a group property: what an aggregate (a summary state -//! or an exact `Aggregate`) covers ([`AggregateCoverage`]): `definition` -//! (what it computes) and `selection` (which output rows of that computation -//! it took, in the IR's selection algebra, `ir::properties::selection`). -//! Design: `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after -//! Goldstein & Larson's view matching. -//! -//! It is derived once for each aggregate memo group from its input groups -//! ([`aggregate_coverage`]), never declared, and stored in its -//! `GroupProperties::aggregate_coverage`. Walking down from a `SummaryAgg` -//! or `Aggregate` through the input groups' expressions, a predicate -//! conjunct moves into `selection` when it can be lifted to the top (through -//! `Filter`, a range `TimeRange`, a `TimeShift` and direct-column `Project` -//! items) and it is a value set or an interval on one column. The -//! `TimeRange` and `TimeShift`s become one interval on the timestamp column -//! ([`IntervalConstraintLimit`]). Everything else stays in `definition` as a -//! residual, so two states are merged only when they compute the same thing -//! over disjoint rows. Node trees carry no coverage. - -use std::collections::BTreeMap; -use std::ops::Bound; - -use thiserror::Error; - -use super::MemoGroupAsInput; -use crate::ir::operator::arguments::{AtModifier, Reduction}; -use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; -use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; -use crate::ir::operator::node::{Operator, OperatorResultKind}; -use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::properties::selection::{ - constraint_of, union, ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, -}; -use crate::ir::scalar::expr_ir::ScalarValue; -use crate::ir::scalar::{Predicate, ScalarExpr}; -use crate::ir::schema::{ColumnId, Schema}; -use crate::ir::SchemaDerivationError; -use crate::stages::MemoGroupId; -use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; - -/// What an aggregate computes, with every lifted selection removed. -#[derive(Debug, Clone, PartialEq)] -pub struct AggregateDefinition { - /// The group the unlifted operators read: the first input under the - /// `SummaryAgg` or exact `Aggregate` that is not a `Filter`, a range - /// `TimeRange`, a `TimeShift` without `@ start()`/`@ end()` or a - /// direct-column `Project`. `None` when that input is a `Scan` with a - /// liftable predicate (a PromQL label matcher): it is lifted too, and the - /// `Scan` with the rest of its predicates is the last unlifted operator, - /// so states over one source with different matchers may merge. - pub input_group: Option, - /// The operators not lifted into the selection, listed top down from the - /// `SummaryAgg` or `Aggregate`, each with only its residual arguments. `None` marks an - /// operator's main input (the next unlifted operator, or input_group); - /// `Some(g)` is another input group it reads, such as the group a scalar - /// subquery in a Filter reads. Two states may merge only if input_group - /// and these operators are equal. - pub unlifted_operators: Vec>>, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct AggregateCoverage { - pub(crate) definition: AggregateDefinition, - /// Union of boxes over the output rows of the unlifted operators' input. - pub(crate) selection: Vec, -} - -#[cfg(any(test, feature = "test-support"))] -impl AggregateCoverage { - pub fn definition(&self) -> &AggregateDefinition { - &self.definition - } - - /// Union of boxes over the output rows of the unlifted operators' input. - pub fn selection(&self) -> &[SelectionBox] { - &self.selection - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum CoverageError { - /// The node has no coverage: it is no aggregate (`SummaryAgg`, - /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a - /// `TimeRange`/`TimeShift` over a chain whose windows are not one - /// interval on the timestamp column. A caller bug, or bad input when it - /// is a merge's input. - #[error("coverage is derived only for aggregates and one time interval over a chain")] - NoCoverage, - /// A `SummaryMerge` without inputs: bad input. - #[error("summary merge requires at least one input")] - EmptyMerge, - /// The inputs of a `SummaryMerge` compute different things: bad input. - #[error("summary merge inputs compute different things")] - DefinitionMismatch, - /// The inputs of a `SummaryMerge` are not proven to relate as the - /// family's merge requires (disjoint, or disjoint in time for counter - /// states): bad input. - #[error( - "summary merge inputs are not proven {}, which the family's merge requires", - match required { - MergeInputRule::DisjointInTime => "disjoint in time", - _ => "disjoint", - } - )] - InputsOverlap { required: MergeInputRule }, - /// A regrouping `SummaryMerge` key that is not one of the group keys - /// its input states are built by (a build grouped `without` keys has - /// none): bad input. - #[error("regrouping merge key {key} is not a group key of its input states")] - NotAGroupKey { key: ColumnId }, - /// A regrouping `SummaryMerge` of states whose summary type does not - /// merge different groups: counter states (`Rate`, `Increase`) merge - /// consecutive windows of one series, a summary type without a merge - /// merges nothing, and a Hydra's groups share one state. Bad input. - #[error("the inputs' summary type cannot merge different groups")] - SummaryTypeCannotRegroup, - /// The build a regrouping `SummaryMerge` stands for does not derive: - /// bad input. - #[error("the regrouped build does not derive: {reason}")] - RegroupedBuild { reason: String }, -} - -/// Coverage of what a memo group computes with `operator` over its -/// input groups: a `SummaryAgg`, an exact `Aggregate`, a -/// `SummaryTumblingWindowChain` (a build over its input, the timestamp -/// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the -/// chain's coverage with the windows' interval on the timestamp column), -/// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped -/// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` -/// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge -/// reads its inputs' stored coverage and fails -/// unless every input has the same definition and their selections are -/// pairwise disjoint; a family whose merge is idempotent -/// ([`MergeInputRule::MayOverlap`] in the family-capability table) may -/// also merge overlapping selections, and a counter family -/// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. -pub(super) fn aggregate_coverage<'m>( - operator: &Operator>, -) -> Result { - let as_built = |op: &ASAPOp>| -> ASAPOp> { - op.map_children(|input| InputOrTemporaryBuild::Input(*input)) - }; - match operator { - Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { - let top = - unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); - of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) - } - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - if operator.output_kind() == OperatorResultKind::State => - { - of_window_selection(operator) - } - Operator::NonASAP(_) => match selected_build(operator)? { - Some(build) => of_summary_agg(&build), - None => Err(CoverageError::NoCoverage), - }, - Operator::ASAP( - op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), - ) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryMerge { - children, group_by, .. - }) => { - let inputs = children - .iter() - .map(|child| { - child - .aggregate_coverage() - .cloned() - .ok_or(CoverageError::NoCoverage) - }) - .collect::, _>>()?; - let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; - match group_by { - None => Ok(merged), - Some(keys) => regrouped(merged, &children[0], keys), - } - } - Operator::ASAP(_) => Err(CoverageError::NoCoverage), - } -} - -/// A memo group read as an input, and the inputs a rule builds over such -/// groups. -type GroupInput<'m> = InputOrTemporaryBuild>; - -/// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by -/// their key columns, the build that computes exactly the kept states: the -/// same build over its input rows filtered by the same predicate on the -/// input columns those keys name. `None` for any other operator. -fn selected_build<'m>( - operator: &Operator>, -) -> Result>>, CoverageError> { - let Operator::NonASAP(NonASAPOp::Filter { - pred, - child: states, - }) = operator - else { - return Ok(None); - }; - if operator.output_kind() != OperatorResultKind::State { - return Ok(None); - } - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction: Reduction::Reduce(by), - filter, - }) = states.operator() - else { - return Err(CoverageError::NoCoverage); - }; - if by.is_without() { - return Err(CoverageError::NoCoverage); - } - // Each state key column is the input column of the same name it groups by. - let mut unmapped = None; - let rows_pred = pred.0.map_columns(&mut |column| { - let mapped = by.keys().get(column).copied().filter(|&input_column| { - matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), - (Some(at_state), Some(at_input)) if at_state.name == at_input.name) - }); - mapped.unwrap_or_else(|| { - unmapped = Some(column); - column - }) - }); - if let Some(key) = unmapped { - return Err(CoverageError::NotAGroupKey { key }); - } - let mut built = |input: &MemoGroupAsInput<'m>| InputOrTemporaryBuild::Input(*input); - let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { - reason: error.to_string(), - }; - let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { - pred: Predicate(rows_pred.map_operator_refs(&mut built)), - child: InputOrTemporaryBuild::Input(child), - }) - .map_err(regrouped_build)?; - let reduction = Reduction::Reduce(by); - summary_agg_layout(rows.schema(), &family, &reduction).map_err(regrouped_build)?; - Ok(Some(ASAPOp::SummaryAgg { - child: rows, - summary_type: family, - input, - reduction, - filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), - })) -} - -/// The coverage of a merge of states covering `inputs`, whose builds read -/// rows with timestamp column `time_column`. -fn of_merge( - inputs: &[AggregateCoverage], - time_column: Option<&ColumnIdentity>, -) -> Result { - let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; - if inputs - .iter() - .any(|input| input.definition != first.definition) - { - return Err(CoverageError::DefinitionMismatch); - } - // A family without a merge is checked as the strictest rule; the - // executor rejects the merge itself. - let required = match first.definition.unlifted_operators.first() { - Some(Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - .. - })) => SummaryTypeCapability::of(family).merge_inputs, - _ => None, - } - .unwrap_or(MergeInputRule::Disjoint); - let separated = |a: &SelectionBox, b: &SelectionBox| match required { - MergeInputRule::MayOverlap => true, - // Consecutive windows of one series: disjoint on the timestamp column. - MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { - matches!( - (a.columns.get(column), b.columns.get(column)), - (Some(a), Some(b)) if a.disjoint(b) - ) - }), - MergeInputRule::Disjoint => a.disjoint(b), - }; - // Every pair of boxes of different inputs. - if required != MergeInputRule::MayOverlap - && inputs.iter().enumerate().any(|(i, input)| { - inputs[..i].iter().any(|other| { - input - .selection - .iter() - .any(|a| other.selection.iter().any(|b| !separated(a, b))) - }) - }) - { - return Err(CoverageError::InputsOverlap { required }); - } - Ok(AggregateCoverage { - definition: first.definition.clone(), - selection: union(inputs.iter().flat_map(|c| c.selection.clone()).collect()), - }) -} - -/// The coverage of a regrouping merge, keyed by `keys`, of states whose -/// merge covers `merged` (`first` is its first input): the same rows -/// (`selection`), computed by the input's build grouped by the columns -/// `keys` name. The input states of one output group are different groups -/// of one build, so their rows are disjoint. A counter family merges only -/// time-disjoint windows, never different groups at one time. -fn regrouped( - merged: AggregateCoverage, - first: &MemoGroupAsInput<'_>, - keys: &[ColumnId], -) -> Result { - let Some(Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - input, - reduction, - filter, - .. - })) = merged.definition.unlifted_operators.first() - else { - return Err(CoverageError::NoCoverage); - }; - if !SummaryTypeCapability::merges_groups(family) { - return Err(CoverageError::SummaryTypeCannotRegroup); - } - let state = first.schema(); - let rows = build_input_schema(first).ok_or(CoverageError::NoCoverage)?; - // The build's input column each key of the state names. - let mapped = keys - .iter() - .map(|&key| { - let column = match reduction { - Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), - // A per-series state keeps its input's columns in place. - Reduction::PerEntity => Some(key), - Reduction::Reduce(_) => None, - }; - column - .filter(|&column| { - let (Some(at_state), Some(at_input)) = - (state.fields.get(key), rows.fields.get(column)) - else { - return false; - }; - at_state.is_plain() - && at_state.name == at_input.name - && at_state.table == at_input.table - }) - .ok_or(CoverageError::NotAGroupKey { key }) - }) - .collect::, _>>()?; - let reduction = Reduction::by(mapped); - summary_agg_layout(&rows, family, &reduction).map_err(|error| { - CoverageError::RegroupedBuild { - reason: error.to_string(), - } - })?; - // A direct build's definition, so it is `==` to one. - let mut definition = merged.definition.clone(); - definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction, - filter: filter.clone(), - }); - Ok(AggregateCoverage { - definition, - selection: merged.selection, - }) -} - -/// The schema of the rows the build of summary state `state` reads (its -/// `SummaryAgg`'s input; one build over a chain's window reads its range of -/// the chain's input, which has the input's schema). `None` when `state` -/// is no summary state. -fn build_input_schema(state: &I) -> Option { - if let Some(states) = selected_from(state) { - return build_input_schema(&states); - } - match state.operator() { - Operator::ASAP( - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => Some(child.schema().clone()), - Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { - build_input_schema(children.first()?) - } - _ => None, - } -} - -/// The timestamp column of the rows the build of summary state `state` -/// reads ([`build_input_schema`]), as a selection names it. -fn time_column_of_build_input(state: &I) -> Option { - let rows = build_input_schema(state)?; - column_identity(&rows, rows.time_index?) -} - -/// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` -/// over memo groups (or over an input a rule built over them): a chain -/// covers what its `SummaryAgg` over the same input would. -fn of_summary_agg(build: &ASAPOp>) -> Result { - let (ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction, - filter, - } - | ASAPOp::SummaryTumblingWindowChain { - child, - summary_type: family, - input, - reduction, - filter, - .. - }) = build - else { - unreachable!("called on a SummaryAgg or a chain"); - }; - of_aggregate(child, filter.as_ref(), |residual_filter| { - Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - filter: residual_filter, - }) - }) -} - -/// The coverage of an aggregate over `child`: its input's operators walked -/// down through the liftable operators of the input groups' expressions. -/// `filter` is the aggregate's own row filter (a `SummaryAgg`'s), and -/// `unlifted_top` makes the aggregate's unlifted operator from that filter's residual. -fn of_aggregate( - child: &GroupInput<'_>, - filter: Option<&Predicate>>, - unlifted_top: impl FnOnce(Option>>) -> Operator>, -) -> Result { - // The operators a predicate can be lifted through, top to bottom, and - // the first input below them. A subquery's group stays named in the - // residual of the operator that reads it. - let mut chain: Vec>> = Vec::new(); - let mut base = child.clone(); - loop { - let operator = base.operator(); - let passes = match &operator { - Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, - Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, - // `@ end()` / `@ start()` pin their subtree to one end of the - // request: not lifted, so their group is the input group. - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - !matches!(shift.at, Some(AtModifier::Start | AtModifier::End)) - } - _ => false, - }; - if !passes { - break; - } - // The main input is an operator's first child. - base = operator.children()[0].clone(); - chain.push(operator); - } - let InputOrTemporaryBuild::Input(input_group) = &base else { - unreachable!("a rule builds only liftable operators over memo groups"); - }; - let top = child.schema(); - // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual - // of `chain[d]` when it is a `Filter`. - let mut lifted = BTreeMap::new(); - let agg_filter = filter.and_then(|pred| residual(pred, top, &[], &mut lifted)); - let kept: Vec>>> = chain - .iter() - .enumerate() - .map(|(depth, link)| match link { - Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { - residual(pred, top, &chain[..depth], &mut lifted) - } - _ => None, - }) - .collect(); - // A `Scan`'s predicates (PromQL label matchers) are lifted too; when - // one is, the `Scan` with the rest of its predicates is the last - // unlifted operator, and there is no input group. - let scan = match base.operator() { - Operator::NonASAP(NonASAPOp::Scan { - source, - predicates, - schema, - }) => { - let lifted_above = lifted.clone(); - let kept = predicates - .iter() - .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) - .collect(); - (lifted != lifted_above).then_some(NonASAPOp::Scan { - source, - predicates: kept, - schema, - }) - } - _ => None, - }; - - // The `TimeRange` and `TimeShift`s: one interval on the timestamp column - // of the `TimeRange`'s input, when they make one and it combines with - // what is lifted on that column already; else they stay unlifted. - let time = time_constraint(&chain).and_then(|constraint| { - let depth = chain - .iter() - .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; - let column = chain[depth].children()[0].schema().time_index?; - let column = column_at_top(column, &chain[..=depth])?; - column_identity(top, column)?; - Some((column, combined_with(lifted.get(&column), constraint)?)) +//! Memo scope of aggregate coverage: the IR derives it +//! ([`crate::ir::properties::aggregate_coverage`]) over memo groups +//! (`R` = [`MemoGroupId`]) and the memo stores it per group +//! (`GroupProperties::aggregate_coverage`); here the plan's view of it, a +//! content id that names no group ([`view_coverage`]). + +use serde::Serialize; + +use crate::ir::Operator; +use crate::plan::physical_asap_dag::ViewCoverage; +use crate::stages::{Memo, MemoGroupId}; + +/// The plan-scope coverage of `group`: the content id of its expanded +/// definition and its selection; `None` for a group without aggregate +/// coverage. The definition's input groups are expanded to their +/// expressions recursively, down to the `Scan`s, so the id names no memo +/// group: two memos (or processes) that compute the same thing give the +/// same id, and different computations different ids. It is the 128-bit +/// FNV-1a hash, in hex, of the expanded definition's JSON (non-finite +/// floats serialized by `serde_f64`). Stage 3 sets it on a materialized +/// view's plan node (`PhysicalASAPDAGNode::view_coverage`). +pub fn view_coverage(memo: &Memo, group: MemoGroupId) -> Option { + let coverage = memo.groups()[group.0 as usize] + .group_properties + .aggregate_coverage + .clone()?; + let expanded = coverage.map_references(|input| expanded(memo, input)); + let serialized = + serde_json::to_string(&expanded.definition).expect("an expanded definition serializes"); + Some(ViewCoverage { + definition_id: fnv1a_128_hex(&serialized), + selection: expanded.selection, + }) +} + +/// What a memo group computes with no ids: its expression over its input +/// groups' expressions, expanded the same way down to the `Scan`s. +#[derive(Serialize)] +#[serde(transparent)] +struct ExpandedExpression(Box>); + +fn expanded(memo: &Memo, group: MemoGroupId) -> ExpandedExpression { + ExpandedExpression(Box::new( + memo.groups()[group.0 as usize] + .expression + .map_children(|&input| expanded(memo, input)), + )) +} + +/// The 128-bit FNV-1a hash of `text`, in hex: stable across processes and +/// Rust versions. +fn fnv1a_128_hex(text: &str) -> String { + const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; + const PRIME: u128 = 0x0000000001000000000000000000013b; + let hash = text.bytes().fold(OFFSET, |hash, byte| { + (hash ^ u128::from(byte)).wrapping_mul(PRIME) }); - let lift_time = time.is_some(); - lifted.extend(time); - - let mut unlifted_operators = vec![unlifted_top(agg_filter)]; - for (depth, link) in chain.iter().enumerate() { - match link { - Operator::NonASAP(NonASAPOp::Filter { .. }) => { - if let Some(rest) = kept[depth].clone() { - unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { - pred: rest, - child: None, - })); - } - } - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - if lift_time => {} - other => unlifted_operators.push(unlifted(other)), - } - } - let input_group = match scan { - Some(scan) => { - unlifted_operators.push(Operator::NonASAP(scan)); - None - } - None => Some(input_group.group_id), - }; - - let selection = SelectionBox { - columns: lifted - .into_iter() - .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) - .collect(), - }; - Ok(AggregateCoverage { - definition: AggregateDefinition { - input_group, - unlifted_operators, - }, - selection: vec![selection], - }) -} - -/// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest, each scalar subquery -/// naming the group it reads. `above` -/// are the operators between the predicate and the `SummaryAgg`, top to -/// bottom. A column [`column_identity`] cannot name stays. -fn residual( - pred: &Predicate>, - top: &Schema, - above: &[Operator>], - lifted: &mut BTreeMap, -) -> Option>> { - let mut rest: Vec>> = Vec::new(); - for conjunct in pred.0.conjuncts() { - let lift = constraint_of(conjunct).and_then(|(column, constraint)| { - let column = column_at_top(column, above)?; - column_identity(top, column)?; - Some((column, combined_with(lifted.get(&column), constraint)?)) - }); - match lift { - Some((column, constraint)) => { - lifted.insert(column, constraint); - } - None => rest.push(conjunct.map_operator_refs(&mut group_id)), - } - } - match rest.len() { - 0 => None, - 1 => rest.pop().map(Predicate), - _ => Some(Predicate(ScalarExpr::BoolAnd(rest))), - } -} - -/// `constraint` combined with what is lifted on its column already -/// (`existing`); `None` when the two do not combine. -fn combined_with(existing: Option<&Constraint>, constraint: Constraint) -> Option { - match existing { - Some(existing) => existing.intersect(&constraint), - None => Some(constraint), - } -} - -/// The memo group `input` is; `None` for a rule's temporary build. -fn group_id(input: &GroupInput<'_>) -> Option { - match input { - InputOrTemporaryBuild::Input(group) => Some(group.group_id), - InputOrTemporaryBuild::TemporaryBuild(_) => None, - } -} - -/// `link`, a liftable operator or an exact `Aggregate` kept whole in a -/// definition: its main input `None`, every other input the group it is. -fn unlifted(link: &Operator>) -> Operator> { - let mut operator = link.map_children(group_id); - match &mut operator { - Operator::NonASAP( - NonASAPOp::Filter { child, .. } - | NonASAPOp::Project { child, .. } - | NonASAPOp::TimeRange { child, .. } - | NonASAPOp::TimeShift { child, .. } - | NonASAPOp::Aggregate { child, .. }, - ) => *child = None, - _ => unreachable!("only liftable operators and aggregates are unlifted"), - } - operator -} - -/// Map a column of the input of `above.last()` up to the agg child's output: -/// a `Project` passes it only as a direct column item. -fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { - for link in above.iter().rev() { - if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { - column = cols - .iter() - .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; - } - } - Some(column) -} - -/// The identity of `schema`'s column `column`; `None` when its -/// `(table, name)` is not unique in `schema`, so a selection cannot name it. -fn column_identity(schema: &Schema, column: ColumnId) -> Option { - let field = schema.fields.get(column)?; - let namesakes = schema - .fields - .iter() - .filter(|f| f.name == field.name && f.table == field.table) - .count(); - (namesakes == 1).then(|| ColumnIdentity { - table: field.table.clone(), - name: field.name.clone(), - }) -} - -/// The interval on the timestamp column that `links` (top down, the -/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: -/// `(anchor − o − W, anchor − o]` for the one range `TimeRange` of width W -/// and the shifts' offsets summing to o, left-open and right-closed like a -/// PromQL range. The anchor is the evaluation time t -/// (`EvaluationTimePlusMs`), or T for `@ T` (`FixedValue(Timestamp)`). -/// `None` when they are not one interval: no or several range `TimeRange`s, -/// an instant one, several `@`s, or an `@ end()` or `@ start()`, which pins -/// its subtree to one end of the request and is never lifted (the front -/// ends put it above the subtree it anchors). -fn time_constraint(links: &[Operator]) -> Option { - let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); - for link in links { - match link { - Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { - if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { - return None; - } - } - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - offset_ms = offset_ms.checked_add(shift.offset_ms)?; - if let Some(anchor) = shift.at { - if at.replace(anchor).is_some() { - return None; - } - } - } - _ => {} - } - } - let width_ms = i64::try_from(width?.as_millis()).ok()?; - let fixed_anchor = match at { - Some(AtModifier::Timestamp(ms)) => Some(ms), - Some(AtModifier::Start | AtModifier::End) => return None, - None => None, - }; - let before_anchor = |ms: i64| -> Option { - Some(match fixed_anchor { - Some(anchor) => { - IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) - } - None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), - }) - }; - Some(Constraint::Interval { - lower: Bound::Excluded(before_anchor(offset_ms.checked_add(width_ms)?)?), - upper: Bound::Included(before_anchor(offset_ms)?), - }) -} - -/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain -/// (through more of them): the chain's coverage with the interval they -/// select on the timestamp column of the chain's input, in every box. -fn of_window_selection( - selection: &Operator>, -) -> Result { - let mut links = vec![selection.clone()]; - let mut below = *selection.children()[0]; - while let Some(child) = selected_from(&below) { - links.push(below.operator()); - below = child; - } - let only_time_selections = links.iter().all(|link| { - matches!( - link, - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - ) - }); - if !only_time_selections - || !matches!( - below.operator(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - { - return Err(CoverageError::NoCoverage); - } - let chain = below - .aggregate_coverage() - .ok_or(CoverageError::NoCoverage)?; - let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NoCoverage)?; - let constraint = time_constraint(&links).ok_or(CoverageError::NoCoverage)?; - let selection = chain - .selection - .iter() - .map(|selection_box| { - let mut columns = selection_box.columns.clone(); - let combined = combined_with(columns.get(&time_column), constraint.clone()) - .ok_or(CoverageError::NoCoverage)?; - columns.insert(time_column.clone(), combined); - Ok(SelectionBox { columns }) - }) - .collect::>()?; - Ok(AggregateCoverage { - definition: chain.definition.clone(), - selection, - }) + format!("{hash:032x}") } diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 2b4f410d6..86039fc38 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -3,10 +3,10 @@ use std::collections::BTreeSet; -use super::aggregate_coverage::{aggregate_coverage, AggregateCoverage}; use super::column_properties::{column_signs, ValueSign}; use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; use crate::ir::operator::input::OperatorInput; +use crate::ir::properties::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; @@ -75,7 +75,7 @@ pub struct GroupProperties { /// What an aggregate group covers: a summary state, a chain, a /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact /// `Aggregate` result; `None` for any other group. - pub aggregate_coverage: Option, + pub aggregate_coverage: Option>, /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). pub shareable_when_equal: bool, @@ -91,12 +91,12 @@ pub struct GroupProperties { /// derived. pub(super) fn validate_expression_over_input_groups( operator: &Operator>, -) -> Result<(Schema, Option), SchemaDerivationError> { +) -> Result<(Schema, Option>), SchemaDerivationError> { let schema = operator.output_schema()?; operator.validate_inputs()?; // A merge is valid only over states of one definition it may merge. let merge_coverage = match operator { - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(aggregate_coverage(operator)?), + Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(operator.aggregate_coverage()?), _ => None, }; Ok((schema, merge_coverage)) @@ -112,7 +112,7 @@ pub(super) fn derive_group_properties( ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); let (schema, merge_coverage) = validate_expression_over_input_groups(&operator)?; - let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(&operator).ok()); + let aggregate_coverage = merge_coverage.or_else(|| operator.aggregate_coverage().ok()); Ok(GroupProperties { column_signs: column_signs(&operator, &schema, &memo.source_facts), depends_on_evaluation_time: depends_on_evaluation_time(&operator), @@ -192,7 +192,13 @@ impl OperatorInput for MemoGroupAsInput<'_> { }) } - fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { + type Reference = MemoGroupId; + + fn reference(&self) -> Option { + Some(self.group_id) + } + + fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { self.group().group_properties.aggregate_coverage.as_ref() } diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 234b6ddc7..59ba58c11 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -10,7 +10,7 @@ mod memo; mod memo_candidates; mod requirements; -pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; +pub use aggregate_coverage::view_coverage; pub use candidate::{ CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index df594f8e5..a0385e635 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -13,11 +13,10 @@ mod physical; mod pruning; pub use logical::{ - AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, - CandidateForMemoGroup, CandidateForMemoGroupId, CoverageError, GroupProperties, - LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, - LogicalRequirements, Memo, MemoGroup, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, - RequiredAccuracy, ValueSign, + view_coverage, CandidateError, CandidateExpression, CandidateForMemoGroup, + CandidateForMemoGroupId, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, + LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupId, + MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, }; pub use per_computation_logical_memo_candidates::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index 31b919290..7b57b5a6f 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -14,16 +14,16 @@ use asap_types::ir::operator::arguments::{AtModifier, Reduction}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::selection::{ColumnIdentity, Constraint, IntervalConstraintLimit}; +use asap_types::ir::properties::{AggregateCoverage, CoverageError}; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, SchemaDerivationError, }; -use asap_types::stages::{AggregateCoverage, CoverageError}; use asap_types::stages::{ - CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, - MemoGroupId, OriginOfCandidateForMemoGroup, + view_coverage, CandidateExpression, CandidateForMemoGroup, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, }; use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; @@ -112,7 +112,7 @@ fn kll_of_last_minute_where( memo.find_or_add_group(kll(last_minute)).unwrap() } -fn coverage(memo: &Memo, group: MemoGroupId) -> &AggregateCoverage { +fn coverage(memo: &Memo, group: MemoGroupId) -> &AggregateCoverage { memo.groups()[group.0 as usize] .group_properties .aggregate_coverage @@ -167,7 +167,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); let coverage = coverage(&memo, g4); - assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!(coverage.definition().input, Some(g1)); assert_eq!( coverage.definition().unlifted_operators, vec![kll_unlifted()] @@ -196,7 +196,7 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { ); let merged = memo.find_or_add_group(merge(vec![g4, g7])).unwrap(); let coverage = coverage(&memo, merged); - assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!(coverage.definition().input, Some(g1)); let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; @@ -238,7 +238,7 @@ fn a_filter_that_is_no_selection_stays_unlifted() { ScalarExpr::BoolAnd(vec![job_is("api"), a_plus_b_above_10()]), ); let coverage_of_g10 = coverage(&memo, g10); - assert_eq!(coverage_of_g10.definition().input_group, Some(g1)); + assert_eq!(coverage_of_g10.definition().input, Some(g1)); assert_eq!( coverage_of_g10.definition().unlifted_operators, vec![ @@ -288,7 +288,7 @@ fn states_whose_definitions_read_the_same_subquery_merge() { let api = state_of_job(&mut memo, "api"); let web = state_of_job(&mut memo, "web"); let definition = coverage(&memo, api).definition(); - assert_eq!(definition.input_group, Some(g1)); + assert_eq!(definition.input, Some(g1)); assert_eq!( definition.unlifted_operators, vec![ @@ -321,7 +321,7 @@ fn scan_predicates_lift_and_their_states_merge() { let api = state_of_job(&mut memo, "api"); let web = state_of_job(&mut memo, "web"); let definition = coverage(&memo, api).definition(); - assert_eq!(definition.input_group, None); + assert_eq!(definition.input, None); assert_eq!( definition.unlifted_operators, vec![kll_unlifted(), latency(vec![]).map_children(|_| None)] @@ -457,7 +457,7 @@ fn at_end_and_at_start_share_the_subtree_they_anchor_and_stay_unlifted() { ); let state = memo.find_or_add_group(kll(pinned)).unwrap(); let coverage = coverage(&memo, state); - assert_eq!(coverage.definition().input_group, Some(pinned)); + assert_eq!(coverage.definition().input, Some(pinned)); assert_eq!(coverage.definition().unlifted_operators, [kll_unlifted()]); assert!(coverage.selection()[0].columns().is_empty()); } @@ -570,7 +570,7 @@ fn an_exact_aggregate_has_coverage() { let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); let g4 = memo.find_or_add_group(sum_by_job(g3)).unwrap(); let coverage = coverage(&memo, g4); - assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!(coverage.definition().input, Some(g1)); assert_eq!( coverage.definition().unlifted_operators, vec![sum_by_job(MemoGroupId(0)).map_children(|_| None)] @@ -584,3 +584,63 @@ fn an_exact_aggregate_has_coverage() { ); assert_eq!(selection.columns()[&ts()], last_minute()); } + +/// KLL over the last minute of `metric{job="api"}` (`scan` → `filter` → +/// `range` → `kll`), in `memo`: its scan group and its state group. +fn kll_of_last_minute_of(memo: &mut Memo, metric: &str) -> (MemoGroupId, MemoGroupId) { + let scan = memo + .find_or_add_group(scan( + metric, + &[ + ("job", DataType::Utf8), + ("a", DataType::Float64), + ("b", DataType::Float64), + ], + vec![], + )) + .unwrap(); + (scan, kll_of_last_minute_where(memo, scan, job_is("api"))) +} + +/// The plan's view of a state's coverage names no memo group: the same +/// state in two memos whose `Scan` groups have different ids has one +/// `definition_id` and one selection. +#[test] +fn a_view_coverage_is_equal_across_memos_whatever_the_group_ids() { + let mut first = Memo::new(&DataWorkload::default()); + let (first_scan, first_state) = kll_of_last_minute_of(&mut first, "latency"); + let mut second = Memo::new(&DataWorkload::default()); + // An unrelated query first, so `latency`'s scan gets another id. + kll_of_last_minute_of(&mut second, "requests"); + let (second_scan, second_state) = kll_of_last_minute_of(&mut second, "latency"); + assert_ne!(first_scan, second_scan); + assert_eq!( + coverage(&first, first_state).definition().input, + Some(first_scan) + ); + let first_view = view_coverage(&first, first_state).unwrap(); + let second_view = view_coverage(&second, second_state).unwrap(); + assert_eq!(first_view.definition_id(), second_view.definition_id()); + assert_eq!(first_view.selection(), second_view.selection()); +} + +/// The same shape over another source computes something else: another +/// `definition_id`. +#[test] +fn a_view_coverage_over_another_source_has_another_definition_id() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, over_latency) = kll_of_last_minute_of(&mut memo, "latency"); + let (_, over_requests) = kll_of_last_minute_of(&mut memo, "requests"); + assert_ne!( + view_coverage(&memo, over_latency).unwrap().definition_id(), + view_coverage(&memo, over_requests).unwrap().definition_id() + ); +} + +/// A group without aggregate coverage (a scan) has no view coverage. +#[test] +fn a_group_without_aggregate_coverage_has_no_view_coverage() { + let mut memo = Memo::new(&DataWorkload::default()); + let (scan, _) = kll_of_last_minute_of(&mut memo, "latency"); + assert_eq!(view_coverage(&memo, scan), None); +} diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index a0d8e8ea8..597811fd6 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -667,7 +667,7 @@ The deployment matches a view to stored states by its coverage coverages by `AggregateDefinition` `==` (its definitions hold memo group ids) and needs no id; the plan carries the view's content id on its node (`PhysicalASAPDAGNode::view_coverage`), defined at -[`ViewCoverage::definition_id`](../../crates/types/src/plan/materialized_view.rs). +[`ViewCoverage::definition_id`](../../crates/types/src/plan/physical_asap_dag.rs). At execution time it checks their actual coverage, revisions and readiness. A compiled plan cannot establish future readiness. The deployment does not replace `NativeKllMerge`, choose another @@ -688,7 +688,7 @@ sketch, or maintain different windows: such changes require replanning. Mapping the materialized-views extension's terms: its `query()` is the sub-DAG under the view's node, and `coverage.definition_id` (the content id, -[`ViewCoverage::definition_id`](../../crates/types/src/plan/materialized_view.rs)) +[`ViewCoverage::definition_id`](../../crates/types/src/plan/physical_asap_dag.rs)) identifies it. Its readiness is the summary store's status, which the deployment owns. Its `mv_dependencies` build graph is `MaterializedView.inputs`. diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 2201ecb76..cfa260d4c 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -117,7 +117,7 @@ A node in the physical data will represent the data or summary instance, so a no | How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | | How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `GroupProperties::aggregate_coverage`, `AggregateCoverage::derive_from_inputs` | §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, `Operator::aggregate_coverage` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -237,35 +237,39 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), `AggregateCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::aggregate_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). +Coverage is an IR property, derived like an output schema and never declared: `Operator::aggregate_coverage` (`crates/types/src/ir/properties/aggregate_coverage.rs`) walks down from a `SummaryAgg` or `Aggregate` through its inputs' operators, generic over how a definition names an input (`OperatorInput::Reference`). The memo derives it once, when `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), over the input groups' `expression`s (`R` = `MemoGroupId`), and stores it in the group's `GroupProperties::aggregate_coverage`; node trees store none. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). ```rust -/// What a summary state computes, with every lifted selection removed. -pub struct AggregateDefinition { - /// The group the unlifted operators read: the first input under the +/// What a summary state computes, with every lifted selection removed; +/// `R` names an input (a memo group's id in the memo). +pub struct AggregateDefinition { + /// The input the unlifted operators read: the first input under the /// SummaryAgg or Aggregate that is not Filter / range TimeRange / /// TimeShift / direct-column Project. `None` when that input is a `Scan` with a /// liftable predicate (a PromQL label matcher): it is lifted too, and the /// `Scan` with the rest of its predicates is the last unlifted operator. - pub input_group: Option, + pub input: Option, /// The operators not lifted into the selection, listed top down from the /// SummaryAgg, each with only its residual arguments. `None` marks an - /// operator's main input (the next unlifted operator, or input_group); - /// `Some(g)` is another input group it reads, such as the group a scalar + /// operator's main input (the next unlifted operator, or `input`); + /// `Some(r)` is another input it reads, such as the input a scalar /// subquery in a Filter reads. - pub unlifted_operators: Vec>>, + pub unlifted_operators: Vec>>, } -pub struct AggregateCoverage { - pub(crate) definition: AggregateDefinition, +pub struct AggregateCoverage { + pub(crate) definition: AggregateDefinition, /// Union of boxes over the output rows of the unlifted operators' input. pub(crate) selection: Vec, } -impl AggregateCoverage { - pub(crate) fn derive_from_inputs( - operator: &Operator>, - ) -> Result; +impl AggregateCoverage { + /// The same coverage with each input reference `r` replaced by `f(r)`. + pub fn map_references(self, f: impl FnMut(R) -> S) -> AggregateCoverage; +} + +impl Operator { + pub fn aggregate_coverage(&self) -> Result, CoverageError>; } ``` @@ -285,26 +289,26 @@ G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryA | 1 | G4 | `SummaryAgg[KLL](G3)` | the first unlifted operator, with no filter of its own | | 2 | G3 | `TimeRange[1m](G2)` | the one range window: lifted as `ts ∈ (t − 60 s, t]` (t the evaluation time) | | 3 | G2 | `Filter(job = "api")(G1)` | `job = "api"` is a value set on one column: lifted, nothing of the `Filter` remains | -| 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input_group` | +| 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input` | ```text G4.aggregate_coverage = AggregateCoverage { definition: AggregateDefinition { - input_group: Some(G1), + input: Some(G1), unlifted_operators: [SummaryAgg[KLL]], }, selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], } ``` -**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) (`==`); its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) (`==`); its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. **Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: ```text G10.aggregate_coverage = AggregateCoverage { definition: AggregateDefinition { - input_group: Some(G1), + input: Some(G1), unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], }, selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], @@ -315,10 +319,10 @@ Its definition differs from G4's, so G10 cannot merge with G4, even though both **The fields.** -- `input_group`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). -- `unlifted_operators`: what the state computes over `input_group`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. +- `input`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). +- `unlifted_operators`: what the state computes over `input`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. - `selection`: which output rows of that computation the state took, as a union of boxes. -- Identity, in two scopes. In the memo, coverages compare by `AggregateDefinition` `==` (it holds `MemoGroupId`s); nothing in the memo needs an id. The plan carries the view's content id, [`ViewCoverage::definition_id`](../../../crates/types/src/plan/materialized_view.rs) (defined there), on the view's plan node (`PhysicalASAPDAGNode::view_coverage`); a summary store files states under it together with a concrete selection. +- Identity, in two scopes. In the memo, coverages compare by `AggregateDefinition` `==` (it holds `MemoGroupId`s); nothing in the memo needs an id. The plan carries the view's content id, [`ViewCoverage::definition_id`](../../../crates/types/src/plan/physical_asap_dag.rs) (defined there), on the view's plan node (`PhysicalASAPDAGNode::view_coverage`), made by `view_coverage(memo, group)`, which expands the memo's coverage (`map_references` to each input group's expression, down to the `Scan`s) rather than re-deriving it; a summary store files states under it together with a concrete selection. ##### The definition @@ -332,8 +336,8 @@ The `definition` is what remains of the operators the walk passed, with every co | the one range `TimeRange`, and every `TimeShift` (an offset or `@ T`; not `@ start()` or `@ end()`, which stay unlifted), when they become one interval on the timestamp column | removed | | a `TimeShift` with no range `TimeRange`, two or more range `TimeRange`s, two `@`s, or an interval that does not combine with a timestamp condition already lifted | unchanged: no time is taken out | | a direct-column `Project` | unchanged | -| a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input_group` is `None` | -| any other operator (the first group the walk cannot pass) | not in the list: it is the `input_group` | +| a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input` is `None` | +| any other operator (the first group the walk cannot pass) | not in the list: it is the `input` | An exact `Aggregate` is the first unlifted operator, kept whole (its per-measure filters and `HAVING` read its input or output, not the rows it reads). A `SummaryTumblingWindowChain[w](X)` has the definition of a build over X, and its selection leaves the timestamp column unconstrained: a chain has no range. A `TimeRange` or `TimeShift` over a chain (a group whose output is the chain's state with a time index) has the chain's coverage with the interval they select on the timestamp column of X, computed as over raw rows. A `SummaryMerge` over that group reads it, so `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` covers exactly what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does, and `Memo::add_candidate` accepts the merge as a candidate of the direct build's group: @@ -347,7 +351,7 @@ G4 candidate: SummaryMerge(G7) the sam A rollup (`SummaryMerge` with `group_by`) has its input's definition with the `SummaryAgg` grouped by the rollup's keys. -**When two definitions are equal.** Merging (§5.6) requires equal definitions: the same `input_group` and the same `unlifted_operators`, compared as values. A definition with a scalar subquery compares like any other: two states whose `Filter(a > (SELECT threshold))` read the same group are equal there. Timing is not part of a group, so a tumbling window built at ingestion time can merge with one built at query time. States over different tables never merge: a KLL over `m1` and one over `m2` read different groups. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. +**When two definitions are equal.** Merging (§5.6) requires equal definitions: the same `input` and the same `unlifted_operators`, compared as values. A definition with a scalar subquery compares like any other: two states whose `Filter(a > (SELECT threshold))` read the same group are equal there. Timing is not part of a group, so a tumbling window built at ingestion time can merge with one built at query time. States over different tables never merge: a KLL over `m1` and one over `m2` read different groups. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. ##### The selection @@ -433,13 +437,13 @@ Any other shape stays in `definition`: - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. -**Time.** The timestamp column is a column like any other: a selection constrains it with an interval whose ends are `IntervalConstraintLimit`s, `EvaluationTimePlusMs(ms)` (relative to the evaluation time t) or `FixedValue(Timestamp(ms since the epoch))` (absolute). The walk turns the one range `TimeRange(W)` and the `TimeShift`s on its path, whose offsets sum to o, into `ts ∈ (anchor − o − W, anchor − o]` on the timestamp column of the `TimeRange`'s input. Like a PromQL range, which `m[r]` evaluated at t reads as (t − r, t], the interval is left-open and right-closed. The anchor is t, or T for `@ T`. `@ end()` and `@ start()` pin a subtree to one end of the request: the PromQL front end lowers the subtree X without them and puts `TimeShift { at: End | Start }(X)` above it, so X is the group of the same expression without `@`. That `TimeShift` is not lifted: the walk stops at it, and it is the definition's `input_group`. Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`; its KLL state has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = `ts ∈ (t − 3m, t − 2m]`. +**Time.** The timestamp column is a column like any other: a selection constrains it with an interval whose ends are `IntervalConstraintLimit`s, `EvaluationTimePlusMs(ms)` (relative to the evaluation time t) or `FixedValue(Timestamp(ms since the epoch))` (absolute). The walk turns the one range `TimeRange(W)` and the `TimeShift`s on its path, whose offsets sum to o, into `ts ∈ (anchor − o − W, anchor − o]` on the timestamp column of the `TimeRange`'s input. Like a PromQL range, which `m[r]` evaluated at t reads as (t − r, t], the interval is left-open and right-closed. The anchor is t, or T for `@ T`. `@ end()` and `@ start()` pin a subtree to one end of the request: the PromQL front end lowers the subtree X without them and puts `TimeShift { at: End | Start }(X)` above it, so X is the group of the same expression without `@`. That `TimeShift` is not lifted: the walk stops at it, and it is the definition's `input`. Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`; its KLL state has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = `ts ∈ (t − 3m, t − 2m]`. | Query fragment | `lower` | `upper` | |---|---|---| | PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | | PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | -| PromQL `m[5m] @ end()` or `@ start()` | the subtree `m[5m]` is the group of `m[5m]`; the `TimeShift { at: End \| Start }` above it is not lifted (it is `input_group`) | | +| PromQL `m[5m] @ end()` or `@ start()` | the subtree `m[5m]` is the group of `m[5m]`; the `TimeShift { at: End \| Start }` above it is not lifted (it is `input`) | | | PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | | SQL `WHERE ts >= now() - INTERVAL '5 minutes'` | lowers to `TimeRange(5m)`: as `m[5m]` | | | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(1_793_491_200_000)))` | `Unbounded` | @@ -950,7 +954,7 @@ OperatorNode §6.1 └── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 - └── definition: AggregateDefinition { input_group, unlifted_operators } + └── definition: AggregateDefinition { input, unlifted_operators } ``` | Subsection | Defines | Used by | @@ -1371,15 +1375,15 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`) +### 6.6 Aggregate coverage (`crates/types/src/ir/properties/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; the plan's content id in `crates/types/src/stages/logical/aggregate_coverage.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. ```rust -pub struct AggregateCoverage { - /// What the state computes: its input group and the operators the walk - /// did not lift (§4.2.2). - pub(crate) definition: AggregateDefinition, +pub struct AggregateCoverage { + /// What the state computes: its input and the operators the walk did + /// not lift (§4.2.2). + pub(crate) definition: AggregateDefinition, /// Which rows went in: a union of boxes. A row is in the state if it is /// in at least one box. pub(crate) selection: Vec, @@ -1435,9 +1439,9 @@ A PromQL range `m[r]` evaluated at t reads the samples in (t − r, t]: left-ope | SQL `WHERE price > 10` | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | ```rust -impl AggregateCoverage { - /// The coverage of a summary-state group computed by `operator` over its - /// input groups (§4.2.2). Errors: `NoCoverage` for an operator that is +impl Operator { + /// The coverage of what this operator computes over its inputs + /// (§4.2.2). Errors: `NoCoverage` for an operator that is /// not a `SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge` or a /// `Filter` keeping whole states; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` /// that does not expand to one build over its whole window; for a @@ -1449,10 +1453,14 @@ impl AggregateCoverage { /// not a group key of the inputs' build), `FamilyCannotRegroup` (counter /// states, a family without a merge, or a Hydra) or `RegroupedBuild` /// (the coarse build it stands for does not derive). - pub(crate) fn derive_from_inputs( - operator: &Operator>, - ) -> Result; + pub fn aggregate_coverage(&self) -> Result, CoverageError>; } + +/// The plan-scope coverage of `group`: the content id of its expanded +/// definition and its selection; `None` for a group without aggregate +/// coverage. Each input group is expanded to its expression, recursively +/// down to the `Scan`s (`map_references`), so the id names no group. +pub fn view_coverage(memo: &Memo, group: MemoGroupId) -> Option; ``` - `find_or_add_group` and `add_candidate` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap, per the family's `MergeInputRule`). Operator-node trees have no coverage and no merge check.