diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index bef383ab9..7ddbe3ba4 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -109,8 +109,8 @@ use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; use asap_types::ir::QueryRoot; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::stages::PhysicalASAPMemoCandidates; -use asap_types::stages::{queries_rooted_at, MemoGroup, MemoGroupId}; use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; +use asap_types::stages::{MemoGroup, MemoGroupId}; use asap_types::workload::{ DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, @@ -757,7 +757,7 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) "children": group.expression.children().into_iter().map(|g| g.0).collect::>(), "parents": ids(group.parent_groups()), "queries": group.reached_by_queries(), - "roots_of": queries_rooted_at(&memo.queries, MemoGroupId(id as u32)), + "roots_of": memo.queries_rooted_at(MemoGroupId(id as u32)), "node": node, "candidate_count": candidates.len(), "candidate_count_by_origin": by_origin, diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index d3e422865..160d792db 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -201,7 +201,7 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi todo!("#1207 PR 9: print each candidate's expression, logical properties and input requirements") } let roots: Vec = memo - .queries + .queries() .iter() .map(|query| match &query.root { QueryRoot::Operator(group) => format!("G{}", group.0), diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index 415b7c0f9..1d1a2eeab 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -71,6 +71,7 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 628737eab..b7ca69528 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -61,6 +61,7 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 8ceb8bdb2..0733da866 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -164,6 +164,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let build = compile_node(&node, std::slice::from_ref(&input)).unwrap(); let wire = serde_json::to_value(&build).unwrap(); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 9bc4f7fdd..517067f4e 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -537,6 +537,7 @@ fn bind_physical_asap_dag_before_execution() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let mut dag = PhysicalASAPDAG { nodes: vec![ @@ -699,6 +700,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer, role, schema: &asap_executor::values::SchemaRef| { PhysicalASAPDAGEdge::new( @@ -1120,6 +1122,7 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -1222,6 +1225,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { right: 1, }), selected_physical_candidate: None, + view_coverage: None, }; let join = if certified { Operator::certified_semi_join(schema.clone(), schema.clone(), vec![(0, 0)]).unwrap() @@ -1329,6 +1333,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { rhs: 1, }), selected_physical_candidate: None, + view_coverage: None, }; let binary = compile_node(&node, &[input.clone(), input.clone()]).unwrap(); for (right, expected) in [ diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index eb395598c..a9b0ccc02 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -391,6 +391,7 @@ fn global_extrema_bind_with_planner_derived_schema() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let operator = compile_node(&node, std::slice::from_ref(&input)) .expect("global extremum should bind to its Planner schema"); @@ -935,6 +936,7 @@ fn planner_node( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } @@ -1168,6 +1170,7 @@ fn exact_aggregate_node( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 6f1aa506d..64f64962f 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -55,6 +55,7 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; PhysicalASAPDAG { nodes: vec![ diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 2ad42fbf7..e9af0acad 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -53,6 +53,7 @@ fn view_over_samples(family: FieldDataType, scan: usize) -> [PhysicalASAPDAGNode guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; [ PhysicalASAPDAGNode { @@ -141,6 +142,7 @@ fn try_state_dag( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); } let read_id = nodes.len(); @@ -156,6 +158,7 @@ fn try_state_dag( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); if let Some(target) = target { nodes.push(PhysicalASAPDAGNode { @@ -176,6 +179,7 @@ fn try_state_dag( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); } let root = nodes.last().unwrap().id; diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 99d260de4..29c23131b 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -87,6 +87,7 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; compile_node(&node, &[schema.clone(), schema]).unwrap() } @@ -428,6 +429,7 @@ fn stored_series_evaluations_support_filters_and_sets() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }) .collect::>(); let edges = [ diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 9edc47cf9..830b799be 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -91,6 +91,7 @@ fn plan( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -513,6 +514,7 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -848,6 +850,7 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } }; dag.nodes.push(node( diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index cc8019aa3..f5391cf28 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -323,6 +323,7 @@ fn merge_over_a_stored_tumbling_window( guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let nodes = vec![ node( diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 3e14c8b2e..1ab25998e 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -58,6 +58,7 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, } } @@ -130,6 +131,7 @@ fn summary_projection_survives_recovery() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }, raw_node(2, scan()), raw_node( @@ -160,6 +162,7 @@ fn summary_projection_survives_recovery() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }, ], edges: vec![ @@ -269,6 +272,7 @@ fn summary_merge_without_inputs_is_rejected() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 2ff8822dd..341977372 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -185,6 +185,7 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }; let build = compile_node(&node, std::slice::from_ref(&input))?; let batch = Batch::try_new( diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 8701574f3..6123568cb 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_stage0_memo`) can match +//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index ea3d3b442..79010baa4 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -487,11 +487,21 @@ fn resolve_expr_in( S::Not(e) => ScalarExpr::Not(bx(e)?), S::IsNull(e) => ScalarExpr::IsNull(bx(e)?), S::IsNotNull(e) => ScalarExpr::IsNotNull(bx(e)?), - S::Cast { expr, to, try_cast } => ScalarExpr::Cast { - expr: bx(expr)?, - to: to.clone(), - try_cast: *try_cast, - }, + // A cast to the type its operand already has is the operand: SQL's + // `ts >= '2026-11-01'` casts the Timestamp column `ts` to DataFusion's + // nanosecond timestamp, which is the same IR type. + S::Cast { expr, to, try_cast } => { + let operand = bx(expr)?; + if matches!(operand.scalar_type(schema), Ok((dtype, _)) if dtype == *to) { + *operand + } else { + ScalarExpr::Cast { + expr: operand, + to: to.clone(), + try_cast: *try_cast, + } + } + } S::InList { expr, list, diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index fa90b507d..f85eaaf4d 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -117,11 +117,12 @@ mod tests { use std::rc::Rc; use std::time::Duration; + use asap_types::ir::operator::arguments::{AtModifier, TimeShift}; use asap_types::ir::operator::Source; - use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; + use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, PlanningWorkload, Query, QueryRequirements, - QueryWorkload, SourceFacts, WorkloadError, + QueryWorkload, RangeEvaluation, SourceFacts, WorkloadError, }; use super::*; @@ -366,4 +367,46 @@ mod tests { ) .is_empty()); } + + /// `@ end()` and `@ start()` are lifted above the selector they anchor, + /// which is lowered without them (its offset stays): the selector of + /// `sum_over_time(data[5m] @ start() offset 1m)` is + /// `TimeShift(@ start())` over the selector of + /// `sum_over_time(data[5m] offset 1m)`, so the two share it. Instant and + /// range-evaluated queries lower alike. + #[test] + fn at_start_and_at_end_are_lifted_above_the_selector_they_anchor() { + let lowered = |query: &str| { + let roots = lower_promql_query_workload(&workload(query)).unwrap(); + let [QueryRoot::Operator(root)] = &roots[..] else { + panic!("one operator root"); + }; + Rc::clone(root.children()[0]) + }; + let pinned = lowered("sum_over_time(data[5m] @ start() offset 1m)"); + let NonASAPOp::TimeShift { shift, child } = pinned.expect_non_asap() else { + panic!("expected the lifted TimeShift, got {pinned:?}"); + }; + assert_eq!( + *shift, + TimeShift { + offset_ms: 0, + at: Some(AtModifier::Start), + } + ); + assert_eq!(*child, lowered("sum_over_time(data[5m] offset 1m)")); + + let mut range_query = workload("sum_over_time(data[5m] @ start() offset 1m)"); + if let Some(batch) = range_query.query_workload.query_batch.as_mut() { + batch[0].range_evaluation = Some(RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }); + } + assert_eq!( + lower_promql_query_workload(&range_query).unwrap(), + lower_promql_query_workload(&workload("sum_over_time(data[5m] @ start() offset 1m)")) + .unwrap() + ); + } } diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 8970cdff8..942593d70 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -55,7 +55,8 @@ //! | `m{f}` / `m{f}[w]` | `TimeRange{ingestion, Instant, Scan{predicates}}` / `TimeRange{w, Range, Scan}` | //! | `a OP b` | `BinaryOp{vector_match}` | //! | `expr[r:res]` | `PromqlSubquery{r, res}` | -//! | ` offset ` / ` @ `/`start()`/`end()` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | +//! | ` offset ` / ` @ ` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | +//! | ` @ start()`/`end()` (also a subquery's) | `TimeShift{at: Start \| End}` above the selector `X`, lowered without the `@` (its offset stays in `X`), so `X` is the group of the same selector without `@` | use std::collections::BTreeMap; use std::rc::Rc; @@ -234,14 +235,16 @@ impl PromqlLowerer<'_> { }; // `offset`/`@` move the whole subquery, including its step grid. let shift = time_shift(sq.offset.as_ref(), sq.at.as_ref())?; - Ok(if shift.is_identity() { - subquery - } else { - Unresolved::TimeShift { - shift, - child: Rc::new(subquery), + Ok(anchored(shift, |shift| { + if shift.is_identity() { + subquery + } else { + Unresolved::TimeShift { + shift, + child: Rc::new(subquery), + } } - }) + })) } Expr::VectorSelector(vs) => { let (metric, matchers, shift) = vs_parts(vs)?; @@ -249,11 +252,13 @@ impl PromqlLowerer<'_> { } Expr::MatrixSelector(ms) => { let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok(Unresolved::TimeRange { - range: ms.range, - kind: TimeRangeKind::Range, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) + Ok(selector( + metric, + matchers, + shift, + ms.range, + TimeRangeKind::Range, + )) } // Scalar-typed, dispatched above; kept for exhaustiveness. String // literals only appear as function args (`label_replace`, …), so a @@ -1120,11 +1125,13 @@ impl PromqlLowerer<'_> { intent: AggIntent, ) -> Result { let child = match inner.window { - Some(w) => Unresolved::TimeRange { - range: w, - kind: TimeRangeKind::Range, - child: Rc::new(filtered_source(inner.metric, inner.matchers, inner.shift)), - }, + Some(w) => selector( + inner.metric, + inner.matchers, + inner.shift, + w, + TimeRangeKind::Range, + ), None => self.instant_source(inner.metric, inner.matchers, inner.shift)?, }; let reduction = reduction_for(&keys, inner.window.is_some() || intent.is_per_series()); @@ -1150,11 +1157,52 @@ impl PromqlLowerer<'_> { shift: TimeShift, ) -> Result { let interval = self.sample_interval(&metric)?; - Ok(Unresolved::TimeRange { - range: interval, - kind: TimeRangeKind::Instant, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) + Ok(selector( + metric, + matchers, + shift, + interval, + TimeRangeKind::Instant, + )) + } +} + +/// A selector over `metric`: the `TimeRange` of `range` and `kind` over its +/// filtered `Scan`, shifted by `shift` ([`anchored`]). +fn selector( + metric: String, + matchers: Vec, + shift: TimeShift, + range: Duration, + kind: TimeRangeKind, +) -> Unresolved { + anchored(shift, |shift| Unresolved::TimeRange { + range, + kind, + child: Rc::new(filtered_source(metric, matchers, shift)), + }) +} + +/// The subtree `lower` builds with `shift`, its `@ start()` / `@ end()` +/// lifted above it: `lower` gets `shift` without them (only the offset and +/// an `@ ` stay in the subtree), and `TimeShift { at: Start | End }` +/// reads its result. The subtree is then the group of the same expression +/// without `@`, shared with any query over it; in a range query it runs +/// once per request, at the request's end (or start), and in an instant +/// query both are the evaluation time. +fn anchored(shift: TimeShift, lower: impl FnOnce(TimeShift) -> Unresolved) -> Unresolved { + match shift.at { + Some(at @ (AtModifier::Start | AtModifier::End)) => Unresolved::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(at), + }, + child: Rc::new(lower(TimeShift { + offset_ms: shift.offset_ms, + at: None, + })), + }, + _ => lower(shift), } } diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index 4ea487c36..232495f3a 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -40,7 +40,7 @@ use asap_frontend_promql::PromqlError as LoweringError; use asap_types::accuracy::AccuracyTarget; use asap_types::ir::operator::{ AggIntent, AtModifier, BinaryOpKind, PromQLVectorSetOpKind, Reduction, SampleKind, Source, - TimeFunc, + TimeFunc, TimeShift, }; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::schema::DataType; @@ -1039,7 +1039,9 @@ fn offset_modifier_lowers_to_a_time_shift() { fn at_modifier_lowers_to_a_time_shift() { // SEMANTICS (PromQL, issue #40): `@ ` pins the evaluation to an absolute // instant (PromQL seconds → IR milliseconds); `@ start()` / `@ end()` anchor - // to the query range bounds. + // to the query range bounds: a `TimeShift { at }` above the selector, which + // is lowered without the `@`, so it is the selector of the same query + // without `@`. let qe = ok("http_requests_total @ 1609746000"); let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { panic!("expected an ingestion TimeRange"); @@ -1051,24 +1053,27 @@ fn at_modifier_lowers_to_a_time_shift() { assert_eq!(shift.offset_ms, 0); let qe = ok("http_requests_total @ start()"); - let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { - panic!("expected an ingestion TimeRange"); - }; - let NonASAPOp::TimeShift { shift, .. } = child.expect_non_asap() else { - panic!("expected a TimeShift for `@ start()`"); + let NonASAPOp::TimeShift { shift, child } = qe.expect_non_asap() else { + panic!("expected a TimeShift for `@ start()`, got {qe:?}"); }; - assert_eq!(shift.at, Some(AtModifier::Start)); + assert_eq!( + *shift, + TimeShift { + offset_ms: 0, + at: Some(AtModifier::Start) + } + ); + assert_eq!(**child, *ok("http_requests_total")); - // Offset and `@` compose: `@ end() offset 5m` carries both. + // Offset and `@` compose: `@ end() offset 5m` is `@ end()` over the + // selector with `offset 5m`. let qe = ok("http_requests_total @ end() offset 5m"); - let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { - panic!("expected an ingestion TimeRange, got {qe:?}"); - }; - let NonASAPOp::TimeShift { shift, .. } = child.expect_non_asap() else { + let NonASAPOp::TimeShift { shift, child } = qe.expect_non_asap() else { panic!("expected a TimeShift, got {qe:?}"); }; assert_eq!(shift.at, Some(AtModifier::End)); - assert_eq!(shift.offset_ms, 300_000); + assert_eq!(shift.offset_ms, 0); + assert_eq!(**child, *ok("http_requests_total offset 5m")); } #[test] diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs index 7e706c414..56043ec36 100644 --- a/crates/frontend-sql/src/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -1,15 +1,17 @@ use std::rc::Rc; +use datafusion::arrow::datatypes::DataType as ArrowDataType; use datafusion::logical_expr::utils::{split_binary, split_conjunction}; use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; use asap_frontend_common::UnresolvedScalar as Unresolved; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::DataType; use asap_types::ir::ExprSemantics; use crate::error::SqlError as LoweringError; -use super::types::{cast_target, scalar_value_to_asap}; +use super::types::{cast_target, dtype_to_arrow, scalar_value_to_asap}; use super::SqlLowerer; impl SqlLowerer<'_> { @@ -137,17 +139,25 @@ impl SqlLowerer<'_> { Expr::IsNotNull(inner) => Ok(Unresolved::IsNotNull(bx(inner)?)), - // Fold a numeric cast of a literal, as DataFusion's constant folding - // does: type coercion turns `latency < 100` on a DOUBLE column into - // `latency < CAST(100 AS DOUBLE)`, which lowers as `latency < 100.0`. + // Fold a numeric or timestamp cast of a literal, as DataFusion's + // constant folding does: type coercion turns `latency < 100` on a + // DOUBLE column into `latency < CAST(100 AS DOUBLE)`, which lowers + // as `latency < 100.0`, and `ts >= '2026-11-01'` into a cast of the + // string, which lowers as a `Timestamp` literal (ms since the epoch). Expr::Cast(c) if matches!(c.expr.as_ref(), Expr::Literal(v, _) if !v.is_null()) - && (c.field.data_type().is_integer() || c.field.data_type().is_floating()) => + && (c.field.data_type().is_integer() + || c.field.data_type().is_floating() + || matches!(c.field.data_type(), ArrowDataType::Timestamp(..))) => { let Expr::Literal(sv, _) = c.expr.as_ref() else { unreachable!() }; - match sv.cast_to(c.field.data_type()) { + let to = match c.field.data_type() { + ArrowDataType::Timestamp(..) => dtype_to_arrow(&DataType::Timestamp), + other => other.clone(), + }; + match sv.cast_to(&to) { Ok(folded) => scalar_value_to_asap(&folded).map(Unresolved::Literal), Err(_) => Ok(Unresolved::Cast { expr: bx(&c.expr)?, diff --git a/crates/frontend-sql/src/sql/types.rs b/crates/frontend-sql/src/sql/types.rs index cf18399ad..08cb21fa9 100644 --- a/crates/frontend-sql/src/sql/types.rs +++ b/crates/frontend-sql/src/sql/types.rs @@ -53,6 +53,13 @@ pub(super) fn scalar_value_to_asap(sv: &DfScalarValue) -> Result Ok(ScalarValue::Utf8(s.clone())), DfScalarValue::Boolean(Some(b)) => Ok(ScalarValue::Boolean(*b)), + // A timestamp is ms since the epoch, as the executor's columns. + DfScalarValue::TimestampSecond(Some(s), _) => Ok(ScalarValue::Timestamp(s * 1_000)), + DfScalarValue::TimestampMillisecond(Some(ms), _) => Ok(ScalarValue::Timestamp(*ms)), + DfScalarValue::TimestampMicrosecond(Some(us), _) => Ok(ScalarValue::Timestamp(us / 1_000)), + DfScalarValue::TimestampNanosecond(Some(ns), _) => { + Ok(ScalarValue::Timestamp(ns / 1_000_000)) + } // All three of DataFusion's interval scalars land on one canonical // shape; the narrower two simply leave the fields they do not carry // at zero. diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 84829c04b..4ae70b9f9 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -2210,7 +2210,9 @@ async fn lead_in_frame_lowers_to_its_own_kind_not_lead() { /// PromQL's Float64 Unix-seconds `EvalTimestamp`. #[tokio::test] async fn now_in_predicate_lowers_to_current_timestamp() { - // WHERE folds onto Scan.predicates (no explicit Filter node). + // WHERE folds onto Scan.predicates (no explicit Filter node). The + // coercion cast of `now()` to the column's timestamp type is dropped: + // `now()` is already a Timestamp. let qe = lower("SELECT * FROM metrics WHERE ts < NOW()").await; let NonASAPOp::Project { child, .. } = op(&qe) else { panic!("expected Project at root, got {qe:?}"); @@ -2221,7 +2223,7 @@ async fn now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "NOW() must lower to CurrentTimestamp, got {:?}", predicates[0].0 ); @@ -2259,7 +2261,7 @@ async fn clickhouse_now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "now() must lower to CurrentTimestamp, got {:?}", predicates[0].0 ); @@ -3199,3 +3201,35 @@ async fn trailing_window_on_an_aliased_table_and_under_asap_rate() { let (window, _) = trailing_window(below).expect("the WHERE window below it"); assert_eq!(window.as_millis(), 300_000); } + +// An absolute timestamp filter compares the time column with a `Timestamp` +// literal (ms since the epoch), so aggregate coverage lifts it into an +// interval on the timestamp column: the cast of the string folds, and the +// cast of the Timestamp column to DataFusion's nanosecond unit is dropped. +#[tokio::test] +async fn an_absolute_timestamp_filter_compares_the_column_with_a_timestamp_literal() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + for sql in [ + "SELECT COUNT(*) FROM metrics WHERE ts >= '2026-11-01'", + "SELECT COUNT(*) FROM metrics WHERE ts >= TIMESTAMP '2026-11-01 00:00:00'", + ] { + let qe = lower(sql).await; + let scan = OperatorNode::reachable(&qe) + .into_iter() + .find(|node| matches!(node.non_asap(), Some(NonASAPOp::Scan { .. }))) + .unwrap_or_else(|| panic!("no Scan for {sql}")); + let NonASAPOp::Scan { predicates, .. } = op(&scan) else { + unreachable!() + }; + assert_eq!( + predicates.iter().map(|pred| &pred.0).collect::>(), + [&ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Ge, + right: Box::new(ScalarExpr::Literal(ScalarValue::Timestamp(NOV_1_2026_MS))), + semantics: ExprSemantics::Sql, + }], + "{sql}" + ); + } +} diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index a240bcb7c..c933d4857 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -302,7 +302,7 @@ pub mod planning { .iter() .enumerate() .filter(|(group, _)| { - memo.queries[q] + memo.queries()[q] .root .operator_refs() .contains(&&asap_types::stages::MemoGroupId(*group as u32)) diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index dc6fb1350..b58aa2ec4 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -57,15 +57,15 @@ pub use asap_stage3_plan_selection::{ AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, }; pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; -pub use asap_types::ir::properties::summary_coverage::{ +pub use asap_types::ir::properties::selection::{ ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, }; pub use asap_types::ir::properties::ExecutionTiming; pub use asap_types::plan::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, MaterializedViewInputs, - MaterializedViewRefresh, RawDataRetention, RawDataWindow, RefreshWindow, ViewCoverage, + MaterializedViewRefresh, RawDataRetention, RawDataWindow, RefreshWindow, }; -pub use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId}; +pub use asap_types::plan::physical_asap_dag::{PhysicalASAPDAG, PhysicalASAPNodeId, ViewCoverage}; pub use asap_types::summary_type::{SketchAlgorithm, SketchParams, StatisticKind}; pub use asap_types::workload::PlanningWorkload; diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index 5227ca379..8a7b78ddf 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -1,30 +1,409 @@ //! `asap-stage0-memo` — #509 Stage 0: the memo groups of the workload's //! query roots (a [`Memo`] without candidates). Each node of the queries' //! DAGs becomes the group of its expression, found or added by -//! [`Memo::find_or_add_group`], so identical sub-expressions are one group. -//! Stage 1 starts from these groups. +//! [`Memo::find_or_add_group`], so identical sub-expressions, within one +//! query or across queries, are one group. Stage 1 starts from these groups. //! -//! It depends only on `asap-types`, where its output (`Memo`) lives; no -//! other stage crate depends on it (the frontends' tests aside, listed in -//! `crate_graph.rs`), and the planner facade calls it first. +//! It depends only on `asap-types`, where its output (`Memo`) lives and +//! where the one expression equality is; no other stage crate depends on +//! it, and the planner facade calls it first. + +use std::collections::HashMap; +use std::rc::Rc; use asap_types::accuracy::AccuracyTarget; -use asap_types::ir::{QueryRoot, SchemaDerivationError}; -use asap_types::stages::Memo; +use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; +use asap_types::stages::{Memo, MemoGroupId, MemoQuery}; +use asap_types::workload::bounds::{series_read, values_per_result}; use asap_types::workload::DataWorkload; /// Stage 0: the memo groups of the workload's query roots, query `i` /// being `roots[i]` with accuracy target `accuracy_targets[i]` (`None`: /// none), every group without candidates. Each query is added with its -/// $n_q$, the number of values one evaluation returns, which `data` bounds; -/// a `Scan`'s column signs are `data`'s declared ones. Fails with the first -/// node that breaks the node rules over its children's groups -/// ([`Memo::find_or_add_group`]). +/// $n_q$, the number of values one evaluation returns, which `data` bounds +/// ([`values_per_result`]); a `Scan`'s column signs are the ones `data` +/// declares, which the memo holds (`Memo::new`). Fails with the first node +/// whose expression is not valid over its children's groups +/// ([`Memo::find_or_add_group`]); the facade validates the roots first, so +/// that is a frontend bug. pub fn memo_of_queries( roots: &[QueryRoot], accuracy_targets: &[Option], data: &DataWorkload, ) -> Result { - let _ = (roots, accuracy_targets, data); - todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group, each query with its target and n_q") + assert_eq!( + roots.len(), + accuracy_targets.len(), + "one accuracy target per query" + ); + let mut memo = Memo::new(data); + let mut group_of_node = GroupOfNode::new(); + for root in roots { + for node in root.operator_refs() { + group_of(&mut memo, &mut group_of_node, node)?; + } + } + let group = |node: &Rc| group_of_node[&Rc::as_ptr(node)]; + for (root, accuracy_target) in roots.iter().zip(accuracy_targets) { + memo.add_query(MemoQuery { + root: match root { + QueryRoot::Operator(node) => QueryRoot::Operator(group(node)), + QueryRoot::Scalar(expr) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |node| group(node))) + } + }, + accuracy_target: accuracy_target.clone(), + number_of_values_per_query_evaluation: values_per_result( + root, + series_read(root, data), + data, + ), + }); + } + Ok(memo) +} + +/// Each node of the queries' DAGs already visited, by address, to its +/// group: a node reached through several paths is visited once. The roots +/// outlive the map, so no address is reused. +type GroupOfNode = HashMap<*const OperatorNode, MemoGroupId>; + +/// The group of `node`: its operator over its children's groups (each +/// found or added first), found or added by [`Memo::find_or_add_group`]. +fn group_of( + memo: &mut Memo, + group_of_node: &mut GroupOfNode, + node: &Rc, +) -> Result { + if let Some(&group) = group_of_node.get(&Rc::as_ptr(node)) { + return Ok(group); + } + for child in node.children() { + group_of(memo, group_of_node, child)?; + } + let expression = node + .operator + .map_children(|child| group_of_node[&Rc::as_ptr(child)]); + let group = memo.find_or_add_group(expression)?; + group_of_node.insert(Rc::as_ptr(node), group); + Ok(group) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + + use asap_types::accuracy::AccuracyTarget; + use asap_types::ir::operator::agg_intent::AggIntent; + use asap_types::ir::operator::arguments::{BinaryOpKind, Reduction, Source}; + use asap_types::ir::operator::asap::ASAPOp; + use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; + use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; + use asap_types::ir::{BinaryOperator, NonASAPOp, Operator, ScalarExpr}; + use asap_types::stages::ValueSign; + use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + + fn node(op: NonASAPOp) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(op)).unwrap() + } + + /// `[ts, service, value, latency]`, no unique key. + fn scan() -> Rc { + node(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) + } + + fn quantile_agg(by: Vec, col: Option, q: f64) -> Rc { + node(NonASAPOp::Aggregate { + reduction: Reduction::by(by), + measures: vec![AggIntent::Quantile { col, q }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }) + } + + fn compare(lhs: Rc, rhs: Rc) -> Rc { + node(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Compare(CompareOpKind::Eq), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) + } + + /// The memo of `roots`, one query per root, without accuracy targets + /// or data facts. + fn memo_of_roots(roots: &[QueryRoot]) -> Result { + memo_of_queries(roots, &vec![None; roots.len()], &DataWorkload::default()) + } + + /// [`memo_of_roots`] of operator roots. + fn memo_of(roots: Vec>) -> Memo { + let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); + memo_of_roots(&roots).unwrap() + } + + /// Query `query`'s root group. + fn root(memo: &Memo, query: usize) -> MemoGroupId { + match &memo.queries()[query].root { + QueryRoot::Operator(group) => *group, + QueryRoot::Scalar(_) => panic!("expected an operator root"), + } + } + + /// Two queries sharing a sub-expression read one group for it, and + /// every group lists the queries that reach it and the groups that + /// read it. + #[test] + fn two_queries_sharing_a_sub_expression_get_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let p50_vs_p90 = compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.9), + ); + let memo = memo_of(vec![p50, p50_vs_p90]); + // scan, p50, p90, compare. + assert_eq!(memo.groups().len(), 4); + let (scan, p50, compare) = (MemoGroupId(0), root(&memo, 0), root(&memo, 1)); + let p90 = MemoGroupId(2); + assert_eq!( + memo.groups()[compare.0 as usize].expression.children(), + vec![&p50, &p90] + ); + assert_eq!(memo.groups()[scan.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p50.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p90.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[compare.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[scan.0 as usize].parent_groups(), &[p50, p90]); + assert_eq!(memo.groups()[p50.0 as usize].parent_groups(), &[compare]); + assert!(memo + .groups() + .iter() + .all(|group| group.candidates.is_empty())); + } + + /// One query's repeated sub-expression is one group, read twice. + #[test] + fn a_query_s_repeated_sub_expression_is_one_group() { + let memo = memo_of(vec![compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.5), + )]); + assert_eq!(memo.groups().len(), 3); + let p50 = MemoGroupId(1); + assert_eq!( + memo.groups()[root(&memo, 0).0 as usize] + .expression + .children(), + vec![&p50, &p50] + ); + assert_eq!( + memo.groups()[p50.0 as usize].parent_groups(), + &[root(&memo, 0)] + ); + } + + /// A node reached through two paths is one group, visited once. + #[test] + fn an_already_shared_node_is_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let memo = memo_of(vec![compare(Rc::clone(&p50), p50)]); + assert_eq!(memo.groups().len(), 3); + } + + /// Queries that differ only in their accuracy target are one group: the + /// target is the query's ([`MemoQuery::accuracy_target`]), not an + /// argument of its operators. + #[test] + fn queries_that_differ_only_in_their_accuracy_target_are_one_group() { + let roots = [ + QueryRoot::Operator(quantile_agg(vec![1], Some(2), 0.99)), + QueryRoot::Operator(quantile_agg(vec![1], Some(2), 0.99)), + ]; + let targets = [ + Some(AccuracyTarget::max_error(0.01)), + Some(AccuracyTarget::max_error(0.05)), + ]; + let memo = memo_of_queries(&roots, &targets, &DataWorkload::default()).unwrap(); + assert_eq!(root(&memo, 0), root(&memo, 1)); + assert_eq!(memo.groups().len(), 2); + assert_eq!(memo.queries()[0].accuracy_target, targets[0]); + assert_eq!(memo.queries()[1].accuracy_target, targets[1]); + } + + /// `vector(scalar(sum by (service) (m)))`. + fn scalar_of_vector() -> Rc { + let sum_up = node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![1]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }); + node(NonASAPOp::PromqlVectorFromScalar( + ScalarExpr::PromqlScalarFromVector(sum_up), + )) + } + + /// A vector read through `scalar(v)` is a child group like any input, + /// one group across queries, and a scalar query's root reads it. + #[test] + fn a_scalar_referenced_vector_is_one_group_across_queries() { + let roots = vec![ + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(Rc::clone( + scalar_of_vector().children()[0], + ))), + ]; + let memo = memo_of_roots(&roots).unwrap(); + // scan, sum, bridge. + assert_eq!(memo.groups().len(), 3); + assert_eq!(root(&memo, 0), root(&memo, 1)); + let sum = MemoGroupId(1); + assert_eq!(memo.queries()[2].root.operator_refs(), vec![&sum]); + assert_eq!( + memo.groups()[sum.0 as usize].reached_by_queries(), + &[0, 1, 2] + ); + } + + /// `SummaryAgg[DDSketch](scan)`. + fn summary_agg() -> Rc { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: scan(), + summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + })) + .unwrap() + } + + /// Two quantile readouts of one summary are two groups over one state + /// group. + #[test] + fn readouts_share_their_state_group_but_not_each_other() { + let readout = |q| { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: summary_agg(), + query: SketchStatistic::Quantile { q }, + })) + .unwrap() + }; + let memo = memo_of(vec![readout(0.95), readout(0.99)]); + let (p95, p99) = (root(&memo, 0), root(&memo, 1)); + assert_ne!(p95, p99); + assert_eq!( + memo.groups()[p95.0 as usize].expression.children(), + memo.groups()[p99.0 as usize].expression.children() + ); + assert_eq!(memo.groups().len(), 4); + } + + /// A frontend node whose expression is not valid over its children's + /// groups (a readout of raw rows instead of summary state) makes Stage 0 + /// return the error. + #[test] + fn memo_of_queries_returns_the_error_of_a_node_invalid_over_its_input_groups() { + let readout_of_rows = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: scan(), + query: SketchStatistic::Quantile { q: 0.5 }, + })) + .unwrap(); + assert!(memo_of_roots(&[QueryRoot::Operator(readout_of_rows)]).is_err()); + } + + /// `m{…}` at the evaluation time: the latest sample of each series. + fn instant_selector(source: Rc) -> Rc { + node(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(60), + kind: asap_types::ir::TimeRangeKind::Instant, + child: source, + }) + } + + /// Each query enters the memo with $n_q$, the values one evaluation + /// returns, as the data workload bounds it: `m`'s 1000 series for the + /// instant vector `m`, one for the scalar `scalar(sum(m))`, unknown for + /// `m` when no series count is declared. + #[test] + fn each_query_enters_with_the_values_one_evaluation_returns() { + let roots = [ + QueryRoot::Operator(instant_selector(scan())), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(Rc::clone( + scalar_of_vector().children()[0], + ))), + ]; + let declared = DataWorkload { + sources: BTreeMap::from([( + "m".to_owned(), + asap_types::workload::SourceFacts { + series_count: Some(1000), + ..Default::default() + }, + )]), + ..Default::default() + }; + let memo = memo_of_queries(&roots, &[None, None], &declared).unwrap(); + assert_eq!( + memo.queries()[0].number_of_values_per_query_evaluation, + Some(1000) + ); + assert_eq!( + memo.queries()[1].number_of_values_per_query_evaluation, + Some(1) + ); + let undeclared = memo_of_roots(&roots).unwrap(); + assert_eq!( + undeclared.queries()[0].number_of_values_per_query_evaluation, + None + ); + } + + /// `memo_of_queries` passes the data workload's facts to the memo: `m` + /// declares `latency` non-negative, which its `Scan` group's signs read. + #[test] + fn a_scan_reads_the_declared_column_signs() { + let data = DataWorkload { + sources: BTreeMap::from([( + "m".to_owned(), + asap_types::workload::SourceFacts { + column_signs: BTreeMap::from([("latency".to_owned(), ValueSign::NonNegative)]), + ..Default::default() + }, + )]), + ..Default::default() + }; + let memo = memo_of_queries(&[QueryRoot::Operator(scan())], &[None], &data).unwrap(); + use ValueSign::{NonNegative, Unknown}; + // [ts, service, value, latency] + assert_eq!( + memo.groups()[root(&memo, 0).0 as usize] + .group_properties + .column_signs, + [Unknown, Unknown, Unknown, NonNegative] + ); + } } diff --git a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs index d66060f30..f46d15f25 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/computation_targets.rs @@ -348,7 +348,7 @@ pub fn generate_per_computation_candidates( // a group several queries read is one node. let node = |g: &MemoGroupId| Rc::clone(&logical_nodes[g.0 as usize]); let roots: Vec = memo - .queries + .queries() .iter() .map(|query| match &query.root { QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), @@ -461,7 +461,7 @@ pub fn generate_per_computation_candidates( } } add_whole_expression_targets(&mut targets, &logical_nodes); - prune_topk_heaps_over_one_row_per_item(&mut targets, &logical_nodes); + prune_topk_heaps_over_one_row_per_item(memo, &mut targets, &logical_nodes); add_nested_sum_targets(&mut targets, &logical_nodes); close_absorptions(&mut targets); Ok(PerComputationCandidates { @@ -718,13 +718,13 @@ fn add_whole_expression_targets( /// one row per item, so exact is no costlier; a fused whole-expression /// heap ([`add_whole_expression_targets`]) has already moved out of them. fn prune_topk_heaps_over_one_row_per_item( + memo: &Memo, targets: &mut [ComputationTarget], logical_nodes: &[Rc], ) { for target in targets.iter_mut() { - let Some(NonASAPOp::Aggregate { - measures, child, .. - }) = logical_nodes[target.aggregate_group.0 as usize].non_asap() + let Some(NonASAPOp::Aggregate { measures, .. }) = + logical_nodes[target.aggregate_group.0 as usize].non_asap() else { continue; }; @@ -735,8 +735,14 @@ fn prune_topk_heaps_over_one_row_per_item( continue; } // A PromQL top-k ranks an instant vector: one row per series or - // group. (A SQL top-k keeps its heaps.) - if !reads_only_time_series(child) { + // group. (A SQL top-k keeps its heaps.) The aggregate scans what its + // input does. + if !memo.groups()[target.aggregate_group.0 as usize] + .group_properties + .scanned_sources + .iter() + .all(Source::is_time_series) + { continue; } prune_own(&mut target.ways_to_compute, |candidate| { @@ -746,14 +752,6 @@ fn prune_topk_heaps_over_one_row_per_item( } } -/// Whether every scan below `node` reads a time series (PromQL rows). -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node.children().iter().all(|c| reads_only_time_series(c)), - } -} - /// The input, update and partitions of a whole-expression top-k over /// `target`'s inner aggregate, by the keyed-additive rule, or `None` when it /// does not apply. Rows that carry the full series identity rank it as a diff --git a/crates/stage3-plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs index 2410b88a5..6452f0bed 100644 --- a/crates/stage3-plan-selection/src/lib.rs +++ b/crates/stage3-plan-selection/src/lib.rs @@ -28,7 +28,9 @@ pub use report::{ /// Stage 3, plan selection: select one plan among `physical`'s candidates /// (module documentation) under `cost_inputs`, and build it: one /// `PhysicalASAPDAG` node per selected execution, its inputs from the -/// selected edges, each node's `selected_physical_candidate` set. +/// selected edges, each node's `selected_physical_candidate` set and each +/// materialized view node's `view_coverage` from its memo group +/// (`asap_types::stages::view_coverage`). /// `entries[q]` is query `q`'s workload entry; `data` the data workload /// with its bounds. /// @@ -44,5 +46,5 @@ pub fn select_plan( .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; let _ = (physical, entries, data); - todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution") + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution, each materialized view node's view_coverage = asap_types::stages::view_coverage(memo, its group)") } diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index e2a6ef1da..2b6ff8f78 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -170,7 +170,7 @@ impl<'de, C: Deserialize<'de>> Deserialize<'de> for GroupKeys { /// The leaf data source of a `Scan`. The schema itself rides on the /// `Scan.schema` field (built by the schema resolver); `Source` carries only the leaf's /// identity. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] pub enum Source { /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. TimeSeries { metric: String }, @@ -178,6 +178,14 @@ pub enum Source { Table { table_ref: String }, } +impl Source { + /// Whether it is a time series (PromQL rows): a PromQL top-k over rows + /// that scan only time series ranks one row per series. + pub fn is_time_series(&self) -> bool { + matches!(self, Self::TimeSeries { .. }) + } +} + /// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's /// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison /// operator has exactly one representation (and one `Display`) across the IR. diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index fc7f44900..1793cbbd6 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -12,6 +12,7 @@ use super::non_asap::NonASAPOp; use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::arguments::Reduction; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; +use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; use crate::summary_type::SummaryTypeCapability; @@ -285,7 +286,7 @@ impl ASAPOp { summary_type: family, reduction, .. - } => summary_agg_schema(child.schema(), family, reduction)?, + } => summary_agg_layout(child.schema(), family, reduction)?.0, SummaryTumblingWindowChain { child, summary_type: family, @@ -427,7 +428,7 @@ impl ASAPOp { reduction, .. }), - ) => summary_agg_schema(child.schema(), &summary_type, &reduction)?, + ) => summary_agg_layout(child.schema(), &summary_type, &reduction)?.0, (None, _) => input.clone(), } } @@ -513,8 +514,9 @@ impl ASAPOp { } } // Equal schemas cannot tell a KLL over `latency` from one over - // `size`, nor prove the inputs disjoint; summary coverage (#646) - // decides whether a structurally valid merge is semantically valid. + // `size`, nor prove the inputs disjoint; aggregate coverage (in + // the memo) decides whether a structurally valid merge is + // semantically valid. Ok(()) } SummaryEstimate { @@ -721,7 +723,7 @@ fn rolled_up_build( }) .collect::>>()?; let reduction = Reduction::by(mapped); - let schema = summary_agg_schema(child.schema(), &summary_type, &reduction).ok()?; + let (schema, _) = summary_agg_layout(child.schema(), &summary_type, &reduction).ok()?; Some(( ASAPOp::SummaryAgg { child, @@ -735,29 +737,31 @@ fn rolled_up_build( } /// The schema of a `SummaryAgg` (or `SummaryTumblingWindowChain`) of -/// `summary_type` by `reduction` over rows of schema `child`: the reduction's -/// output columns, its value column replaced by the state. -pub(crate) fn summary_agg_schema( +/// `summary_type` by `reduction` over rows of schema `child`, and where each +/// of its columns comes from: the reduction's output columns, its one +/// measure's value column replaced by the state +/// (`OutputColumnSource::Measure`). +pub(crate) fn summary_agg_layout( child: &Schema, summary_type: &FieldDataType, reduction: &Reduction, -) -> Result { - let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( +) -> Result<(Schema, Vec), SchemaDerivationError> { + let (mut schema, sources) = aggregate_output( child, reduction, &[crate::ir::operator::AggIntent::Sum { col: None }], &[], )?; - let index = match reduction { - Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( - &crate::ir::scalar::ColumnRef::SampleValue, - &schema, - ) - .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, - Reduction::Reduce(_) => schema.fields.len() - 1, - }; + let index = sources + .iter() + .position(|source| matches!(source, OutputColumnSource::Measure { .. })) + .ok_or_else(|| { + SchemaDerivationError::InvalidScalarSignature( + "a summary state has no value column".into(), + ) + })?; schema.fields[index] = Field::new("state", summary_type.clone(), false); - Ok(schema) + Ok((schema, sources)) } /// The schema of a `SummaryTumblingWindowChain` of `summary_type` by `reduction` @@ -771,7 +775,7 @@ fn chain_schema( summary_type: &FieldDataType, reduction: &Reduction, ) -> Result { - let mut schema = summary_agg_schema(child, summary_type, reduction)?; + let (mut schema, _) = summary_agg_layout(child, summary_type, reduction)?; let time = child .time_index .and_then(|index| child.fields.get(index)) diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index b5d0ac29d..3cc9a088e 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -4,11 +4,13 @@ //! `GroupProperties` (concepts doc `memo-groups-properties-requirements.md`, //! section 3). +use std::collections::BTreeSet; use std::rc::Rc; +use super::arguments::Source; use super::node::{Operator, OperatorNode, OperatorResultKind}; use super::non_asap::NonASAPOp; -use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::properties::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; @@ -18,12 +20,22 @@ pub trait OperatorInput: Sized + Clone { fn result_kind(&self) -> OperatorResultKind; /// The input's own operator, its children as inputs again. fn operator(&self) -> Operator; - /// What the input covers when it is an aggregate group in the memo; - /// `None` for any other input. - fn aggregate_coverage(&self) -> Option<&SummaryCoverage>; + /// How an aggregate's definition names this input + /// ([`AggregateCoverage`]): a memo group's id in the memo. + type Reference: Clone + PartialEq; + /// This input's reference; `None` for a build a rule constructs only + /// while it derives, which has none. + fn reference(&self) -> Option; + /// What the input covers when its coverage is stored (an aggregate + /// group in the memo); `None` for any other input. + fn aggregate_coverage(&self) -> Option<&AggregateCoverage>; + /// The sources the `Scan`s at or below the input read + /// (`Operator::scanned_sources` of its operator): a memo group's stored + /// `GroupProperties::scanned_sources`, recomputed for a node tree. + fn scanned_sources(&self) -> BTreeSet; } -/// A front-end node tree. Node trees carry no summary coverage. +/// A front-end node tree. Node trees store no aggregate coverage. impl OperatorInput for Rc { fn schema(&self) -> &Schema { &self.schema @@ -37,9 +49,21 @@ impl OperatorInput for Rc { self.operator.clone() } - fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + /// A node tree is named by nothing: no definition over a node tree is + /// stored or compared. + type Reference = (); + + fn reference(&self) -> Option<()> { + Some(()) + } + + fn aggregate_coverage(&self) -> Option<&AggregateCoverage<()>> { None } + + fn scanned_sources(&self) -> BTreeSet { + self.operator.scanned_sources() + } } /// An input of a rule: a real input `I`, or a build the rule constructs @@ -109,10 +133,26 @@ impl OperatorInput for InputOrTemporaryBuild { } } - fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + type Reference = I::Reference; + + fn reference(&self) -> Option { + match self { + Self::Input(input) => input.reference(), + Self::TemporaryBuild(_) => None, + } + } + + fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { match self { Self::Input(input) => input.aggregate_coverage(), Self::TemporaryBuild(_) => None, } } + + fn scanned_sources(&self) -> BTreeSet { + match self { + Self::Input(input) => input.scanned_sources(), + Self::TemporaryBuild(_) => self.operator().scanned_sources(), + } + } } diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 7551f4d6c..72bc8ad8f 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -5,11 +5,12 @@ //! each physical candidate holds its own //! ([`crate::stages::PhysicalCandidate::properties`]). -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; use std::rc::Rc; use serde::{Deserialize, Serialize}; +use super::arguments::Source; use super::asap::ASAPOp; use super::input::OperatorInput; use super::non_asap::NonASAPOp; @@ -65,6 +66,20 @@ impl Operator { /// The derivation rules, one set for every kind of input /// ([`OperatorInput`]): front-end node trees and memo groups. impl Operator { + /// The sources the `Scan`s at or below this operator read: a `Scan`'s + /// own, else the union over its inputs' + /// ([`OperatorInput::scanned_sources`]). + pub(crate) fn scanned_sources(&self) -> BTreeSet { + match self { + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => BTreeSet::from([source.clone()]), + operator => operator + .children() + .into_iter() + .flat_map(OperatorInput::scanned_sources) + .collect(), + } + } + pub(crate) fn output_schema(&self) -> Result { match self { Operator::NonASAP(op) => op.output_schema(), diff --git a/crates/types/src/ir/properties/aggregate_coverage.rs b/crates/types/src/ir/properties/aggregate_coverage.rs new file mode 100644 index 000000000..fbfca1122 --- /dev/null +++ b/crates/types/src/ir/properties/aggregate_coverage.rs @@ -0,0 +1,781 @@ +//! Aggregate coverage, an IR property derived like an output schema: what +//! an aggregate (a summary state or an exact `Aggregate`) covers +//! ([`AggregateCoverage`]): `definition` (what it computes) and `selection` +//! (which output rows of that computation it took, in the selection algebra +//! of [`super::selection`]). Design: +//! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after +//! Goldstein & Larson's view matching. +//! +//! It is derived over an operator's inputs ([`Operator::aggregate_coverage`]), +//! generic over how a definition names an input +//! ([`OperatorInput::Reference`]): the memo derives it once per aggregate +//! group (`R` = `MemoGroupId`) and stores it in +//! `GroupProperties::aggregate_coverage`; a node tree stores none. Walking +//! down from a `SummaryAgg` or `Aggregate` through its inputs' operators, a +//! predicate conjunct moves into `selection` when it can be lifted to the +//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and +//! direct-column `Project` items) and it is a value set or an interval on +//! one column. The `TimeRange` and `TimeShift`s become one interval on the +//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! `definition` as a residual, so two states are merged only when they +//! compute the same thing over disjoint rows. + +use std::collections::BTreeMap; +use std::ops::Bound; + +use thiserror::Error; + +use super::selection::{ + constraint_of, union, ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, +}; +use crate::ir::operator::arguments::{AtModifier, Reduction}; +use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; +use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; +use crate::ir::operator::node::{Operator, OperatorResultKind}; +use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; +use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::scalar::{Predicate, ScalarExpr}; +use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::SchemaDerivationError; +use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; + +/// What an aggregate computes, with every lifted selection removed. +/// `R` is how it names an input: a memo group's id in the memo. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct AggregateDefinition { + /// The input the unlifted operators read: the first input under the + /// `SummaryAgg` or exact `Aggregate` that is not a `Filter`, a range + /// `TimeRange`, a `TimeShift` without `@ start()`/`@ end()` or a + /// direct-column `Project`. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator, + /// so states over one source with different matchers may merge. + pub input: Option, + /// The operators not lifted into the selection, listed top down from the + /// `SummaryAgg` or `Aggregate`, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or `input`); + /// `Some(r)` is another input it reads, such as the input a scalar + /// subquery in a Filter reads. Two states may merge only if `input` + /// and these operators are equal. + pub unlifted_operators: Vec>>, +} + +/// What an aggregate covers, its inputs named by `R`. +#[derive(Debug, Clone, PartialEq)] +pub struct AggregateCoverage { + pub(crate) definition: AggregateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. + pub(crate) selection: Vec, +} + +impl AggregateCoverage { + /// The same coverage with each input reference `r` replaced by `f(r)`. + pub fn map_references(self, mut f: impl FnMut(R) -> S) -> AggregateCoverage { + let AggregateDefinition { + input, + unlifted_operators, + } = self.definition; + AggregateCoverage { + definition: AggregateDefinition { + input: input.map(&mut f), + unlifted_operators: unlifted_operators + .into_iter() + .map(|operator| operator.map_children(|child| child.clone().map(&mut f))) + .collect(), + }, + selection: self.selection, + } + } +} + +#[cfg(any(test, feature = "test-support"))] +impl AggregateCoverage { + pub fn definition(&self) -> &AggregateDefinition { + &self.definition + } + + /// Union of boxes over the output rows of the unlifted operators' input. + pub fn selection(&self) -> &[SelectionBox] { + &self.selection + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CoverageError { + /// The node has no coverage: it is no aggregate (`SummaryAgg`, + /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a + /// `TimeRange`/`TimeShift` over a chain whose windows are not one + /// interval on the timestamp column. A caller bug, or bad input when it + /// is a merge's input. + #[error("coverage is derived only for aggregates and one time interval over a chain")] + NoCoverage, + /// A `SummaryMerge` without inputs: bad input. + #[error("summary merge requires at least one input")] + EmptyMerge, + /// The inputs of a `SummaryMerge` compute different things: bad input. + #[error("summary merge inputs compute different things")] + DefinitionMismatch, + /// The inputs of a `SummaryMerge` are not proven to relate as the + /// family's merge requires (disjoint, or disjoint in time for counter + /// states): bad input. + #[error( + "summary merge inputs are not proven {}, which the family's merge requires", + match required { + MergeInputRule::DisjointInTime => "disjoint in time", + _ => "disjoint", + } + )] + InputsOverlap { required: MergeInputRule }, + /// A regrouping `SummaryMerge` key that is not one of the group keys + /// its input states are built by (a build grouped `without` keys has + /// none): bad input. + #[error("regrouping merge key {key} is not a group key of its input states")] + NotAGroupKey { key: ColumnId }, + /// A regrouping `SummaryMerge` of states whose summary type does not + /// merge different groups: counter states (`Rate`, `Increase`) merge + /// consecutive windows of one series, a summary type without a merge + /// merges nothing, and a Hydra's groups share one state. Bad input. + #[error("the inputs' summary type cannot merge different groups")] + SummaryTypeCannotRegroup, + /// The build a regrouping `SummaryMerge` stands for does not derive: + /// bad input. + #[error("the regrouped build does not derive: {reason}")] + RegroupedBuild { reason: String }, +} + +impl Operator { + /// The coverage of what this operator computes over its inputs + /// (`coverage_of`, below). + pub fn aggregate_coverage(&self) -> Result, CoverageError> { + coverage_of(self) + } +} + +/// Coverage of what `operator` computes over its inputs: a `SummaryAgg`, an exact `Aggregate`, a +/// `SummaryTumblingWindowChain` (a build over its input, the timestamp +/// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the +/// chain's coverage with the windows' interval on the timestamp column), +/// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped +/// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` +/// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge +/// reads its inputs' stored coverage and fails +/// unless every input has the same definition and their selections are +/// pairwise disjoint; a family whose merge is idempotent +/// ([`MergeInputRule::MayOverlap`] in the family-capability table) may +/// also merge overlapping selections, and a counter family +/// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. +fn coverage_of( + operator: &Operator, +) -> Result, CoverageError> { + let as_built = |op: &ASAPOp| -> ASAPOp> { + op.map_children(|input| InputOrTemporaryBuild::Input(input.clone())) + }; + match operator { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = unlifted( + &operator.map_children(|input| InputOrTemporaryBuild::Input(input.clone())), + ); + of_aggregate(&InputOrTemporaryBuild::Input(child.clone()), None, |_| top) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if operator.output_kind() == OperatorResultKind::State => + { + of_window_selection(operator) + } + Operator::NonASAP(_) => match selected_build(operator)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NoCoverage), + }, + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => of_summary_agg(&as_built(op)), + Operator::ASAP(ASAPOp::SummaryMerge { + children, group_by, .. + }) => { + let inputs = children + .iter() + .map(|child| { + child + .aggregate_coverage() + .cloned() + .ok_or(CoverageError::NoCoverage) + }) + .collect::, _>>()?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; + match group_by { + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), + } + } + Operator::ASAP(_) => Err(CoverageError::NoCoverage), + } +} + +/// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by +/// their key columns, the build that computes exactly the kept states: the +/// same build over its input rows filtered by the same predicate on the +/// input columns those keys name. `None` for any other operator. +fn selected_build( + operator: &Operator, +) -> Result>>, CoverageError> { + let Operator::NonASAP(NonASAPOp::Filter { + pred, + child: states, + }) = operator + else { + return Ok(None); + }; + if operator.output_kind() != OperatorResultKind::State { + return Ok(None); + } + let Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: family, + input, + reduction: Reduction::Reduce(by), + filter, + }) = states.operator() + else { + return Err(CoverageError::NoCoverage); + }; + if by.is_without() { + return Err(CoverageError::NoCoverage); + } + // Each state key column is the input column of the same name it groups by. + let mut unmapped = None; + let rows_pred = pred.0.map_columns(&mut |column| { + let mapped = by.keys().get(column).copied().filter(|&input_column| { + matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), + (Some(at_state), Some(at_input)) if at_state.name == at_input.name) + }); + mapped.unwrap_or_else(|| { + unmapped = Some(column); + column + }) + }); + if let Some(key) = unmapped { + return Err(CoverageError::NotAGroupKey { key }); + } + let mut built = |input: &I| InputOrTemporaryBuild::Input(input.clone()); + let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { + reason: error.to_string(), + }; + let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { + pred: Predicate(rows_pred.map_operator_refs(&mut built)), + child: InputOrTemporaryBuild::Input(child), + }) + .map_err(regrouped_build)?; + let reduction = Reduction::Reduce(by); + summary_agg_layout(rows.schema(), &family, &reduction).map_err(regrouped_build)?; + Ok(Some(ASAPOp::SummaryAgg { + child: rows, + summary_type: family, + input, + reduction, + filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), + })) +} + +/// The coverage of a merge of states covering `inputs`, whose builds read +/// rows with timestamp column `time_column`. +fn of_merge( + inputs: &[AggregateCoverage], + time_column: Option<&ColumnIdentity>, +) -> Result, CoverageError> { + let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; + if inputs + .iter() + .any(|input| input.definition != first.definition) + { + return Err(CoverageError::DefinitionMismatch); + } + // A family without a merge is checked as the strictest rule; the + // executor rejects the merge itself. + let required = match first.definition.unlifted_operators.first() { + Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + })) => SummaryTypeCapability::of(family).merge_inputs, + _ => None, + } + .unwrap_or(MergeInputRule::Disjoint); + let separated = |a: &SelectionBox, b: &SelectionBox| match required { + MergeInputRule::MayOverlap => true, + // Consecutive windows of one series: disjoint on the timestamp column. + MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { + matches!( + (a.columns.get(column), b.columns.get(column)), + (Some(a), Some(b)) if a.disjoint(b) + ) + }), + MergeInputRule::Disjoint => a.disjoint(b), + }; + // Every pair of boxes of different inputs. + if required != MergeInputRule::MayOverlap + && inputs.iter().enumerate().any(|(i, input)| { + inputs[..i].iter().any(|other| { + input + .selection + .iter() + .any(|a| other.selection.iter().any(|b| !separated(a, b))) + }) + }) + { + return Err(CoverageError::InputsOverlap { required }); + } + Ok(AggregateCoverage { + definition: first.definition.clone(), + selection: union(inputs.iter().flat_map(|c| c.selection.clone()).collect()), + }) +} + +/// The coverage of a regrouping merge, keyed by `keys`, of states whose +/// merge covers `merged` (`first` is its first input): the same rows +/// (`selection`), computed by the input's build grouped by the columns +/// `keys` name. The input states of one output group are different groups +/// of one build, so their rows are disjoint. A counter family merges only +/// time-disjoint windows, never different groups at one time. +fn regrouped( + merged: AggregateCoverage, + first: &I, + keys: &[ColumnId], +) -> Result, CoverageError> { + let Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + input, + reduction, + filter, + .. + })) = merged.definition.unlifted_operators.first() + else { + return Err(CoverageError::NoCoverage); + }; + if !SummaryTypeCapability::merges_groups(family) { + return Err(CoverageError::SummaryTypeCannotRegroup); + } + let state = first.schema(); + let rows = build_input_schema(first).ok_or(CoverageError::NoCoverage)?; + // The build's input column each key of the state names. + let mapped = keys + .iter() + .map(|&key| { + let column = match reduction { + Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), + // A per-series state keeps its input's columns in place. + Reduction::PerEntity => Some(key), + Reduction::Reduce(_) => None, + }; + column + .filter(|&column| { + let (Some(at_state), Some(at_input)) = + (state.fields.get(key), rows.fields.get(column)) + else { + return false; + }; + at_state.is_plain() + && at_state.name == at_input.name + && at_state.table == at_input.table + }) + .ok_or(CoverageError::NotAGroupKey { key }) + }) + .collect::, _>>()?; + let reduction = Reduction::by(mapped); + summary_agg_layout(&rows, family, &reduction).map_err(|error| { + CoverageError::RegroupedBuild { + reason: error.to_string(), + } + })?; + // A direct build's definition, so it is `==` to one. + let mut definition = merged.definition.clone(); + definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), + input: input.clone(), + reduction, + filter: filter.clone(), + }); + Ok(AggregateCoverage { + definition, + selection: merged.selection, + }) +} + +/// The schema of the rows the build of summary state `state` reads (its +/// `SummaryAgg`'s input; one build over a chain's window reads its range of +/// the chain's input, which has the input's schema). `None` when `state` +/// is no summary state. +fn build_input_schema(state: &I) -> Option { + if let Some(states) = selected_from(state) { + return build_input_schema(&states); + } + match state.operator() { + Operator::ASAP( + ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, + ) => Some(child.schema().clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { + build_input_schema(children.first()?) + } + _ => None, + } +} + +/// The timestamp column of the rows the build of summary state `state` +/// reads ([`build_input_schema`]), as a selection names it. +fn time_column_of_build_input(state: &I) -> Option { + let rows = build_input_schema(state)?; + column_identity(&rows, rows.time_index?) +} + +/// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` +/// over inputs (or over an input a rule built over them): a chain covers +/// what its `SummaryAgg` over the same input would. +fn of_summary_agg( + build: &ASAPOp>, +) -> Result, CoverageError> { + let (ASAPOp::SummaryAgg { + child, + summary_type: family, + input, + reduction, + filter, + } + | ASAPOp::SummaryTumblingWindowChain { + child, + summary_type: family, + input, + reduction, + filter, + .. + }) = build + else { + unreachable!("called on a SummaryAgg or a chain"); + }; + of_aggregate(child, filter.as_ref(), |residual_filter| { + Operator::ASAP(ASAPOp::SummaryAgg { + child: None, + summary_type: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + filter: residual_filter, + }) + }) +} + +/// The coverage of an aggregate over `child`: its input's operators walked +/// down through the liftable operators of its inputs' operators. +/// `filter` is the aggregate's own row filter (a `SummaryAgg`'s), and +/// `unlifted_top` makes the aggregate's unlifted operator from that filter's residual. +fn of_aggregate( + child: &InputOrTemporaryBuild, + filter: Option<&Predicate>>, + unlifted_top: impl FnOnce(Option>>) -> Operator>, +) -> Result, CoverageError> { + // The operators a predicate can be lifted through, top to bottom, and + // the first input below them. A subquery's input stays named in the + // residual of the operator that reads it. + let mut chain: Vec>> = Vec::new(); + let mut base = child.clone(); + loop { + let operator = base.operator(); + let passes = match &operator { + Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, + // `@ end()` / `@ start()` pin their subtree to one end of the + // request: not lifted, so their group is the input group. + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + !matches!(shift.at, Some(AtModifier::Start | AtModifier::End)) + } + _ => false, + }; + if !passes { + break; + } + // The main input is an operator's first child. + base = operator.children()[0].clone(); + chain.push(operator); + } + let InputOrTemporaryBuild::Input(base_input) = &base else { + unreachable!("a rule builds only liftable operators over real inputs"); + }; + let top = child.schema(); + // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual + // of `chain[d]` when it is a `Filter`. + let mut lifted = BTreeMap::new(); + let agg_filter = filter.and_then(|pred| residual(pred, top, &[], &mut lifted)); + let kept: Vec>>> = chain + .iter() + .enumerate() + .map(|(depth, link)| match link { + Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { + residual(pred, top, &chain[..depth], &mut lifted) + } + _ => None, + }) + .collect(); + // A `Scan`'s predicates (PromQL label matchers) are lifted too; when + // one is, the `Scan` with the rest of its predicates is the last + // unlifted operator, and there is no input. + let scan = match base.operator() { + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) => { + let lifted_above = lifted.clone(); + let kept = predicates + .iter() + .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) + .collect(); + (lifted != lifted_above).then_some(NonASAPOp::Scan { + source, + predicates: kept, + schema, + }) + } + _ => None, + }; + + // The `TimeRange` and `TimeShift`s: one interval on the timestamp column + // of the `TimeRange`'s input, when they make one and it combines with + // what is lifted on that column already; else they stay unlifted. + let time = time_constraint(&chain).and_then(|constraint| { + let depth = chain + .iter() + .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; + let column = chain[depth].children()[0].schema().time_index?; + let column = column_at_top(column, &chain[..=depth])?; + column_identity(top, column)?; + Some((column, combined_with(lifted.get(&column), constraint)?)) + }); + let lift_time = time.is_some(); + lifted.extend(time); + + let mut unlifted_operators = vec![unlifted_top(agg_filter)]; + for (depth, link) in chain.iter().enumerate() { + match link { + Operator::NonASAP(NonASAPOp::Filter { .. }) => { + if let Some(rest) = kept[depth].clone() { + unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { + pred: rest, + child: None, + })); + } + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if lift_time => {} + other => unlifted_operators.push(unlifted(other)), + } + } + let input = match scan { + Some(scan) => { + unlifted_operators.push(Operator::NonASAP(scan)); + None + } + None => base_input.reference(), + }; + + let selection = SelectionBox { + columns: lifted + .into_iter() + .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) + .collect(), + }; + Ok(AggregateCoverage { + definition: AggregateDefinition { + input, + unlifted_operators, + }, + selection: vec![selection], + }) +} + +/// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of +/// `top`, the agg child's schema) and return the rest, each scalar subquery +/// naming the input it reads. `above` +/// are the operators between the predicate and the `SummaryAgg`, top to +/// bottom. A column [`column_identity`] cannot name stays. +fn residual( + pred: &Predicate>, + top: &Schema, + above: &[Operator>], + lifted: &mut BTreeMap, +) -> Option>> { + let mut rest: Vec>> = Vec::new(); + for conjunct in pred.0.conjuncts() { + let lift = constraint_of(conjunct).and_then(|(column, constraint)| { + let column = column_at_top(column, above)?; + column_identity(top, column)?; + Some((column, combined_with(lifted.get(&column), constraint)?)) + }); + match lift { + Some((column, constraint)) => { + lifted.insert(column, constraint); + } + None => rest.push(conjunct.map_operator_refs(&mut reference_of)), + } + } + match rest.len() { + 0 => None, + 1 => rest.pop().map(Predicate), + _ => Some(Predicate(ScalarExpr::BoolAnd(rest))), + } +} + +/// `constraint` combined with what is lifted on its column already +/// (`existing`); `None` when the two do not combine. +fn combined_with(existing: Option<&Constraint>, constraint: Constraint) -> Option { + match existing { + Some(existing) => existing.intersect(&constraint), + None => Some(constraint), + } +} + +/// How a definition names `input`; `None` for a rule's temporary build. +fn reference_of(input: &InputOrTemporaryBuild) -> Option { + input.reference() +} + +/// `link`, a liftable operator or an exact `Aggregate` kept whole in a +/// definition: its main input `None`, every other input its reference. +fn unlifted( + link: &Operator>, +) -> Operator> { + let mut operator = link.map_children(reference_of); + match &mut operator { + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::Project { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. } + | NonASAPOp::Aggregate { child, .. }, + ) => *child = None, + _ => unreachable!("only liftable operators and aggregates are unlifted"), + } + operator +} + +/// Map a column of the input of `above.last()` up to the agg child's output: +/// a `Project` passes it only as a direct column item. +fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { + for link in above.iter().rev() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { + column = cols + .iter() + .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; + } + } + Some(column) +} + +/// The identity of `schema`'s column `column`; `None` when its +/// `(table, name)` is not unique in `schema`, so a selection cannot name it. +fn column_identity(schema: &Schema, column: ColumnId) -> Option { + let field = schema.fields.get(column)?; + let namesakes = schema + .fields + .iter() + .filter(|f| f.name == field.name && f.table == field.table) + .count(); + (namesakes == 1).then(|| ColumnIdentity { + table: field.table.clone(), + name: field.name.clone(), + }) +} + +/// The interval on the timestamp column that `links` (top down, the +/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: +/// `(anchor − o − W, anchor − o]` for the one range `TimeRange` of width W +/// and the shifts' offsets summing to o, left-open and right-closed like a +/// PromQL range. The anchor is the evaluation time t +/// (`EvaluationTimePlusMs`), or T for `@ T` (`FixedValue(Timestamp)`). +/// `None` when they are not one interval: no or several range `TimeRange`s, +/// an instant one, several `@`s, or an `@ end()` or `@ start()`, which pins +/// its subtree to one end of the request and is never lifted (the front +/// ends put it above the subtree it anchors). +fn time_constraint(links: &[Operator]) -> Option { + let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); + for link in links { + match link { + Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { + if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { + return None; + } + } + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + offset_ms = offset_ms.checked_add(shift.offset_ms)?; + if let Some(anchor) = shift.at { + if at.replace(anchor).is_some() { + return None; + } + } + } + _ => {} + } + } + let width_ms = i64::try_from(width?.as_millis()).ok()?; + let fixed_anchor = match at { + Some(AtModifier::Timestamp(ms)) => Some(ms), + Some(AtModifier::Start | AtModifier::End) => return None, + None => None, + }; + let before_anchor = |ms: i64| -> Option { + Some(match fixed_anchor { + Some(anchor) => { + IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) + } + None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), + }) + }; + Some(Constraint::Interval { + lower: Bound::Excluded(before_anchor(offset_ms.checked_add(width_ms)?)?), + upper: Bound::Included(before_anchor(offset_ms)?), + }) +} + +/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain +/// (through more of them): the chain's coverage with the interval they +/// select on the timestamp column of the chain's input, in every box. +fn of_window_selection( + selection: &Operator, +) -> Result, CoverageError> { + let mut links = vec![selection.clone()]; + let mut below = selection.children()[0].clone(); + while let Some(child) = selected_from(&below) { + links.push(below.operator()); + below = child; + } + let only_time_selections = links.iter().all(|link| { + matches!( + link, + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + ) + }); + if !only_time_selections + || !matches!( + below.operator(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) + { + return Err(CoverageError::NoCoverage); + } + let chain = below + .aggregate_coverage() + .ok_or(CoverageError::NoCoverage)?; + let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NoCoverage)?; + let constraint = time_constraint(&links).ok_or(CoverageError::NoCoverage)?; + let selection = chain + .selection + .iter() + .map(|selection_box| { + let mut columns = selection_box.columns.clone(); + let combined = combined_with(columns.get(&time_column), constraint.clone()) + .ok_or(CoverageError::NoCoverage)?; + columns.insert(time_column.clone(), combined); + Ok(SelectionBox { columns }) + }) + .collect::>()?; + Ok(AggregateCoverage { + definition: chain.definition.clone(), + selection, + }) +} diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 5eecc5878..dff3b6bd9 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -1,10 +1,18 @@ //! #511 §2.2–2.3: the planning properties a node carries besides its -//! operator and schema. +//! operator and schema (when its value is produced), and what an aggregate +//! covers with the selection algebra it is stated in. //! //! - [`execution_data_state`] — when a node's value is produced //! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). -//! - [`summary_coverage`] — what a summary state covers. +//! - [`aggregate_coverage`] — what an aggregate computes and which rows it +//! took ([`AggregateCoverage`]), derived over an operator's inputs. +//! - [`selection`] — which rows of a computation an aggregate's state took: +//! boxes of per-column constraints, the selection algebra aggregate +//! coverage is built from. +pub mod aggregate_coverage; pub mod execution_data_state; -pub mod summary_coverage; +pub mod selection; + +pub use aggregate_coverage::{AggregateCoverage, AggregateDefinition, CoverageError}; pub use execution_data_state::{DataPrimitive, ExecutionDataState, ExecutionTiming}; diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs new file mode 100644 index 000000000..6bd47b5de --- /dev/null +++ b/crates/types/src/ir/properties/selection.rs @@ -0,0 +1,640 @@ +//! The selection algebra of aggregate coverage +//! ([`AggregateCoverage`](super::AggregateCoverage)'s `selection`): which output rows of a +//! computation a state took, as a union of boxes ([`SelectionBox`]), each a +//! [`Constraint`] per column ([`ColumnIdentity`]): value sets and intervals, +//! with interval limits relative to the evaluation time +//! ([`IntervalConstraintLimit`]). A predicate conjunct becomes a constraint +//! through `constraint_of`. +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::ops::Bound; + +use serde::{Deserialize, Serialize}; + +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::ScalarExpr; +use crate::ir::schema::ColumnId; + +/// A conjunction of per-column constraints. The timestamp column is one of +/// the columns; there is no separate time field. A column it does not +/// mention is unrestricted. JSON: `columns` is a list of +/// `{column, constraint}` (its keys are not strings), and a bound is +/// `{"included": v}`, `{"excluded": v}` or `"unbounded"`. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SelectionBox { + #[serde(with = "columns_as_list")] + pub(crate) columns: BTreeMap, +} + +/// `SelectionBox::columns` as a list of `{column, constraint}`. +mod columns_as_list { + use super::{ColumnIdentity, Constraint}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::collections::BTreeMap; + + #[derive(Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + struct Entry { + column: ColumnIdentity, + constraint: Constraint, + } + + pub(super) fn serialize( + columns: &BTreeMap, + serializer: S, + ) -> Result { + let entries: Vec = columns + .iter() + .map(|(column, constraint)| Entry { + column: column.clone(), + constraint: constraint.clone(), + }) + .collect(); + entries.serialize(serializer) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + let entries = Vec::::deserialize(deserializer)?; + Ok(entries + .into_iter() + .map(|entry| (entry.column, entry.constraint)) + .collect()) + } +} + +/// A `Bound` in JSON: `{"included": v}`, `{"excluded": v}` or `"unbounded"`. +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum WireBound { + Included(T), + Excluded(T), + Unbounded, +} + +impl From<&Bound> for WireBound { + fn from(bound: &Bound) -> Self { + match bound { + Bound::Included(v) => Self::Included(v.clone()), + Bound::Excluded(v) => Self::Excluded(v.clone()), + Bound::Unbounded => Self::Unbounded, + } + } +} + +impl From> for Bound { + fn from(bound: WireBound) -> Self { + match bound { + WireBound::Included(v) => Bound::Included(v), + WireBound::Excluded(v) => Bound::Excluded(v), + WireBound::Unbounded => Bound::Unbounded, + } + } +} + +/// A column of the definition child's output, by `(table, name)`. A +/// `Project` below renames its columns and sets their table to its +/// qualifier (none by default). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ColumnIdentity { + pub(crate) table: Option, + pub(crate) name: String, +} + +#[cfg(any(test, feature = "test-support"))] +impl ColumnIdentity { + pub fn new(table: Option, name: impl Into) -> Self { + Self { + table, + name: name.into(), + } + } +} + +/// A constraint on one column of a selection. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Constraint { + /// The value is one of these. + In(Vec), + /// The value is none of these. + NotIn(Vec), + /// The value lies between `lower` and `upper`; each end is `Included`, + /// `Excluded` or `Unbounded`. A PromQL range `m[r]` evaluated at t reads + /// the samples in (t − r, t]: left-open, right-closed. Examples, on the + /// timestamp column `ts` (ms) unless stated: + /// + /// | Query fragment | `lower` | `upper` | + /// |---|---|---| + /// | PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | + /// | PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | + /// | PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | + /// | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | + /// | SQL `WHERE price > 10` (column `price`) | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | + /// + /// A full-range interval `(Unbounded, Unbounded)` is still a + /// constraint, not the absence of one: it keeps every ordered value and + /// excludes the rows whose value is NULL or NaN. A column with no + /// constraint in a box keeps every row. + Interval { + #[serde(with = "limit_bound")] + lower: Bound, + #[serde(with = "limit_bound")] + upper: Bound, + }, +} + +/// An end of a [`Constraint::Interval`]. Two limits compare only when they +/// are of the same variant: a fixed value against a fixed value of the same +/// type, an offset from the evaluation time against another such offset. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IntervalConstraintLimit { + /// A constant: `Float64(10.0)` for SQL `price > 10`; + /// `Timestamp(1_700_000_000_000)` for PromQL `@ 1700000000` or a SQL + /// timestamp literal. + FixedValue(ScalarValue), + /// The evaluation time t of a repeatedly evaluated query plus this many + /// ms (negative: before t); only on the timestamp column. `m[5m]` is + /// `(EvaluationTimePlusMs(-300_000), EvaluationTimePlusMs(0)]`. + EvaluationTimePlusMs(i64), +} + +/// A `Constraint::Interval` bound as a [`WireBound`]. +mod limit_bound { + use super::{IntervalConstraintLimit, WireBound}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::ops::Bound; + + pub(super) fn serialize( + bound: &Bound, + serializer: S, + ) -> Result { + WireBound::from(bound).serialize(serializer) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + Ok(WireBound::deserialize(deserializer)?.into()) + } +} + +/// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of +/// equalities on one column, with non-null literals. +pub(crate) fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { + let literal = |e: &ScalarExpr| match e { + ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), + _ => None, + }; + match expr { + ScalarExpr::Compare { + left, op, right, .. + } => { + let (column, value, op) = match (left.as_ref(), right.as_ref()) { + (ScalarExpr::Column(c), r) => (*c, literal(r)?, op.clone()), + (l, ScalarExpr::Column(c)) => (*c, literal(l)?, flip(op)?), + _ => return None, + }; + let constraint = match op { + CompareOpKind::Eq => Constraint::In(vec![value]), + CompareOpKind::Ne => Constraint::NotIn(vec![value]), + CompareOpKind::Lt => interval(Bound::Unbounded, Bound::Excluded(value)), + CompareOpKind::Le => interval(Bound::Unbounded, Bound::Included(value)), + CompareOpKind::Gt => interval(Bound::Excluded(value), Bound::Unbounded), + CompareOpKind::Ge => interval(Bound::Included(value), Bound::Unbounded), + _ => return None, + }; + Some((column, constraint)) + } + ScalarExpr::InList { + expr, + list, + negated, + } => { + let ScalarExpr::Column(column) = expr.as_ref() else { + return None; + }; + let values = list.iter().map(literal).collect::>>()?; + let values = dedup(values); + Some(( + *column, + if *negated { + Constraint::NotIn(values) + } else { + Constraint::In(values) + }, + )) + } + ScalarExpr::BoolOr(disjuncts) => { + let mut column = None; + let mut values = Vec::new(); + for disjunct in disjuncts { + let (c, Constraint::In(v)) = constraint_of(disjunct)? else { + return None; + }; + if column.replace(c).is_some_and(|prev| prev != c) { + return None; + } + values.extend(v); + } + Some((column?, Constraint::In(dedup(values)))) + } + _ => None, + } +} + +fn flip(op: &CompareOpKind) -> Option { + Some(match op { + CompareOpKind::Eq => CompareOpKind::Eq, + CompareOpKind::Ne => CompareOpKind::Ne, + CompareOpKind::Lt => CompareOpKind::Gt, + CompareOpKind::Le => CompareOpKind::Ge, + CompareOpKind::Gt => CompareOpKind::Lt, + CompareOpKind::Ge => CompareOpKind::Le, + _ => return None, + }) +} + +/// An interval between two fixed values. +fn interval(lower: Bound, upper: Bound) -> Constraint { + Constraint::Interval { + lower: lower.map(IntervalConstraintLimit::FixedValue), + upper: upper.map(IntervalConstraintLimit::FixedValue), + } +} + +fn dedup(values: Vec) -> Vec { + let mut out: Vec = Vec::new(); + for v in values { + if !out.contains(&v) { + out.push(v); + } + } + out +} + +/// Whether `v` equals one of `values`; `None` when some pair cannot be +/// compared (different types, NaN), since `1` and `1.0` may select the same +/// rows. +fn among(v: &ScalarValue, values: &[ScalarValue]) -> Option { + for w in values { + if compare(v, w)? == Ordering::Equal { + return Some(true); + } + } + Some(false) +} + +/// The values of `a` that are (`keep = true`) or are not in `b`. +fn filter_values(a: &[ScalarValue], b: &[ScalarValue], keep: bool) -> Option> { + let mut out = Vec::new(); + for v in a { + if among(v, b)? == keep { + out.push(v.clone()); + } + } + Some(out) +} + +/// Order of two literals of the same type; `None` across types or for NaN. +fn compare(a: &ScalarValue, b: &ScalarValue) -> Option { + match (a, b) { + (ScalarValue::Int64(a), ScalarValue::Int64(b)) => Some(a.cmp(b)), + (ScalarValue::Float64(a), ScalarValue::Float64(b)) => a.partial_cmp(b), + (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => Some(a.cmp(b)), + (ScalarValue::Boolean(a), ScalarValue::Boolean(b)) => Some(a.cmp(b)), + (ScalarValue::Timestamp(a), ScalarValue::Timestamp(b)) => Some(a.cmp(b)), + _ => None, + } +} + +/// Order of two interval limits; `None` across variants or when the fixed +/// values do not compare. +fn compare_limits(a: &IntervalConstraintLimit, b: &IntervalConstraintLimit) -> Option { + use IntervalConstraintLimit::*; + match (a, b) { + (FixedValue(a), FixedValue(b)) => compare(a, b), + (EvaluationTimePlusMs(a), EvaluationTimePlusMs(b)) => Some(a.cmp(b)), + _ => None, + } +} + +/// Whether value `v` lies inside the interval; `None` when unsure. +fn value_inside( + v: &ScalarValue, + bounds: ( + &Bound, + &Bound, + ), +) -> Option { + inside( + &IntervalConstraintLimit::FixedValue(v.clone()), + bounds, + compare_limits, + ) +} + +/// Whether `v` lies inside the interval; `None` when it cannot be compared. +fn inside( + v: &T, + (lower, upper): (&Bound, &Bound), + cmp: impl Fn(&T, &T) -> Option, +) -> Option { + let above = match lower { + Bound::Unbounded => true, + Bound::Included(l) => cmp(v, l)? != Ordering::Less, + Bound::Excluded(l) => cmp(v, l)? == Ordering::Greater, + }; + let below = match upper { + Bound::Unbounded => true, + Bound::Included(u) => cmp(v, u)? != Ordering::Greater, + Bound::Excluded(u) => cmp(v, u)? == Ordering::Less, + }; + Some(above && below) +} + +/// Whether no value is both at most `upper` and at least `lower`. +fn ends_before( + upper: &Bound, + lower: &Bound, + cmp: impl Fn(&T, &T) -> Option, +) -> bool { + match (upper, lower) { + (Bound::Included(u), Bound::Included(l)) => cmp(u, l) == Some(Ordering::Less), + (Bound::Included(u) | Bound::Excluded(u), Bound::Included(l) | Bound::Excluded(l)) => { + matches!(cmp(u, l), Some(Ordering::Less | Ordering::Equal)) + } + _ => false, + } +} + +fn intervals_disjoint( + (al, au): (&Bound, &Bound), + (bl, bu): (&Bound, &Bound), + cmp: impl Fn(&T, &T) -> Option + Copy, +) -> bool { + ends_before(au, bl, cmp) || ends_before(bu, al, cmp) +} + +/// The tighter of two lower (`want = Greater`) or upper (`Less`) bounds. +fn tighter( + a: &Bound, + b: &Bound, + want: Ordering, +) -> Option> { + let value = |bound: &Bound| match bound { + Bound::Included(v) | Bound::Excluded(v) => Some(v.clone()), + Bound::Unbounded => None, + }; + Some(match (value(a), value(b)) { + (None, _) => b.clone(), + (_, None) => a.clone(), + (Some(x), Some(y)) => match compare_limits(&x, &y)? { + Ordering::Equal if matches!(a, Bound::Excluded(_)) => a.clone(), + Ordering::Equal => b.clone(), + order if order == want => a.clone(), + _ => b.clone(), + }, + }) +} + +impl Constraint { + /// The conjunction of two constraints on one column, when it is again a + /// single constraint. + pub(crate) fn intersect(&self, other: &Self) -> Option { + use Constraint::*; + Some(match (self, other) { + (In(a), In(b)) => In(filter_values(a, b, true)?), + (In(a), NotIn(b)) | (NotIn(b), In(a)) => In(filter_values(a, b, false)?), + (NotIn(a), NotIn(b)) => NotIn(dedup(a.iter().chain(b).cloned().collect())), + (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => { + let mut kept = Vec::new(); + for v in a { + if value_inside(v, (lower, upper))? { + kept.push(v.clone()); + } + } + In(kept) + } + ( + Interval { + lower: al, + upper: au, + }, + Interval { + lower: bl, + upper: bu, + }, + ) => Interval { + lower: tighter(al, bl, Ordering::Greater)?, + upper: tighter(au, bu, Ordering::Less)?, + }, + (NotIn(_), Interval { .. }) | (Interval { .. }, NotIn(_)) => return None, + }) + } + + /// Whether no value satisfies both; `false` when unsure. + pub(crate) fn disjoint(&self, other: &Self) -> bool { + use Constraint::*; + match (self, other) { + (In(a), In(b)) => a.iter().all(|v| among(v, b) == Some(false)), + (In(a), NotIn(b)) | (NotIn(b), In(a)) => a.iter().all(|v| among(v, b) == Some(true)), + (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => a + .iter() + .all(|v| value_inside(v, (lower, upper)) == Some(false)), + ( + Interval { + lower: al, + upper: au, + }, + Interval { + lower: bl, + upper: bu, + }, + ) => intervals_disjoint((al, au), (bl, bu), compare_limits), + (NotIn(_), _) | (_, NotIn(_)) => false, + } + } +} + +#[cfg(any(test, feature = "test-support"))] +impl SelectionBox { + pub fn new(columns: BTreeMap) -> Self { + Self { columns } + } + + /// The per-column constraints. + pub fn columns(&self) -> &BTreeMap { + &self.columns + } +} + +impl SelectionBox { + /// Whether no row lies in both boxes: some column is restricted by both + /// and the restrictions are disjoint. + pub(crate) fn disjoint(&self, other: &Self) -> bool { + self.columns + .iter() + .any(|(column, a)| other.columns.get(column).is_some_and(|b| a.disjoint(b))) + } +} + +/// Join boxes that differ only in one column whose union is again one +/// constraint: touching value ranges, or the values of one `In` column. +/// Gaps stay separate boxes. +pub(crate) fn union(mut boxes: Vec) -> Vec { + let mut i = 0; + while i < boxes.len() { + let joined = (i + 1..boxes.len()).find_map(|j| join(&boxes[i], &boxes[j]).map(|b| (j, b))); + match joined { + Some((j, joined)) => { + boxes.remove(j); + boxes[i] = joined; + i = 0; + } + None => i += 1, + } + } + boxes +} + +fn join(a: &SelectionBox, b: &SelectionBox) -> Option { + if a == b { + return Some(a.clone()); + } + if a.columns.len() != b.columns.len() { + return None; + } + let mut differing = a + .columns + .iter() + .filter(|(column, constraint)| b.columns.get(*column) != Some(*constraint)); + let (column, constraint) = differing.next()?; + if differing.next().is_some() { + return None; + } + let joined = match (constraint, b.columns.get(column)?) { + (Constraint::In(values), Constraint::In(more)) => { + Constraint::In(dedup(values.iter().chain(more).cloned().collect())) + } + ( + Constraint::Interval { + lower: al, + upper: au, + }, + Constraint::Interval { + lower: bl, + upper: bu, + }, + ) => { + let (lower, upper) = touching((al, au), (bl, bu), compare_limits)?; + Constraint::Interval { lower, upper } + } + _ => return None, + }; + let mut columns = a.columns.clone(); + columns.insert(column.clone(), joined); + Some(SelectionBox { columns }) +} + +/// The union of two intervals when one ends exactly where the other starts, +/// with the shared end point in exactly one of them. +fn touching( + (al, au): (&Bound, &Bound), + (bl, bu): (&Bound, &Bound), + cmp: impl Fn(&T, &T) -> Option, +) -> Option<(Bound, Bound)> { + let meets = |upper: &Bound, lower: &Bound| match (upper, lower) { + (Bound::Included(u), Bound::Excluded(l)) | (Bound::Excluded(u), Bound::Included(l)) => { + cmp(u, l) == Some(Ordering::Equal) + } + _ => false, + }; + if meets(au, bl) { + Some((al.clone(), bu.clone())) + } else if meets(bu, al) { + Some((bl.clone(), au.clone())) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::scalar::ExprSemantics; + + use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; + + fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { + Constraint::Interval { + lower: Bound::Excluded(EvaluationTimePlusMs(lower_ms)), + upper: Bound::Included(EvaluationTimePlusMs(upper_ms)), + } + } + + /// `m[5m]` and `m[5m] offset 5m` are disjoint and touch; neither + /// compares with an `@ T` interval. + #[test] + fn limits_compare_only_within_one_variant() { + let last_5m = relative(-300_000, 0); + let offset_5m = relative(-600_000, -300_000); + assert!(last_5m.disjoint(&offset_5m)); + let at = Constraint::Interval { + lower: Bound::Excluded(FixedValue(ScalarValue::Timestamp(1_699_999_700_000))), + upper: Bound::Included(FixedValue(ScalarValue::Timestamp(1_700_000_000_000))), + }; + assert!(!last_5m.disjoint(&at)); + assert_eq!(last_5m.intersect(&at), None); + let column = ColumnIdentity::new(None, "ts"); + let one = |constraint: Constraint| SelectionBox { + columns: BTreeMap::from([(column.clone(), constraint)]), + }; + assert_eq!( + union(vec![one(last_5m), one(offset_5m)]), + vec![one(relative(-600_000, 0))] + ); + } + + #[test] + fn an_interval_round_trips_through_json() { + let constraint = relative(-3_900_000, -3_600_000); + let json = serde_json::to_value(&constraint).unwrap(); + assert_eq!( + json, + serde_json::json!({"interval": { + "lower": {"excluded": {"evaluation_time_plus_ms": -3_900_000}}, + "upper": {"included": {"evaluation_time_plus_ms": -3_600_000}}, + }}) + ); + assert_eq!( + serde_json::from_value::(json).unwrap(), + constraint + ); + } + + /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. + #[test] + fn a_value_comparison_is_an_interval_between_fixed_values() { + let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + }; + assert_eq!( + constraint_of(&price_gt_10), + Some(( + 0, + Constraint::Interval { + lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), + upper: Bound::Unbounded, + } + )) + ); + } +} diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs deleted file mode 100644 index e67a5acc0..000000000 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ /dev/null @@ -1,1210 +0,0 @@ -//! What a summary state covers: `definition` (what it computes) and -//! `selection` (which output rows of that computation it took). Design: -//! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after -//! Goldstein & Larson's view matching. -//! -//! Coverage is derived once for each summary-state memo group, from its -//! input groups, never declared, and stored in its `GroupProperties`. -//! Walking down from a `SummaryAgg` through the input groups' expressions, a -//! predicate conjunct moves into `selection` when it can be lifted to the -//! `SummaryAgg` (through `Filter`, a range `TimeRange`, a `TimeShift` without -//! `@` and direct-column `Project` items) and it is a value set or an -//! interval on one column. Everything else stays in `definition` as a -//! residual, so two states are merged only when they compute the same thing -//! over disjoint rows. Node trees carry no coverage. -use std::cmp::Ordering; -use std::collections::BTreeMap; -use std::ops::Bound; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -use crate::ir::operator::arguments::Reduction; -use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; -use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; -use crate::ir::operator::node::{Operator, OperatorResultKind}; -use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; -use crate::ir::scalar::{Predicate, ScalarExpr}; -use crate::ir::schema::{ColumnId, Schema}; -use crate::ir::SchemaDerivationError; -use crate::stages::{MemoGroupAsInput, MemoGroupId}; -use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; - -/// What a summary state computes, with every lifted selection removed. -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryStateDefinition { - /// The group the unlifted operators read: the first input under the - /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ - /// / direct-column Project. `None` when that input is a `Scan` with a - /// liftable predicate (a PromQL label matcher): it is lifted too, and the - /// `Scan` with the rest of its predicates is the last unlifted operator, - /// so states over one source with different matchers may merge. - pub input_group: Option, - /// The operators not lifted into the selection, listed top down from the - /// SummaryAgg, each with only its residual arguments. `None` marks an - /// operator's main input (the next unlifted operator, or input_group); - /// `Some(g)` is another input group it reads, such as the group a scalar - /// subquery in a Filter reads. Two states may merge only if input_group - /// and these operators are equal. - pub unlifted_operators: Vec>>, -} - -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryCoverage { - pub(crate) definition: SummaryStateDefinition, - /// Union of boxes over the output rows of the unlifted operators' input. - pub(crate) selection: Vec, -} - -#[cfg(any(test, feature = "test-support"))] -impl SummaryCoverage { - pub fn definition(&self) -> &SummaryStateDefinition { - &self.definition - } - - /// Union of boxes over the output rows of the unlifted operators' input. - pub fn selection(&self) -> &[SelectionBox] { - &self.selection - } -} - -/// A conjunction of per-column constraints. The timestamp column is one of -/// the columns; there is no separate time field. A column it does not -/// mention is unrestricted. JSON: `columns` is a list of -/// `{column, constraint}` (its keys are not strings), and a bound is -/// `{"included": v}`, `{"excluded": v}` or `"unbounded"`. -#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SelectionBox { - #[serde(with = "columns_as_list")] - pub(crate) columns: BTreeMap, -} - -/// `SelectionBox::columns` as a list of `{column, constraint}`. -mod columns_as_list { - use super::{ColumnIdentity, Constraint}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use std::collections::BTreeMap; - - #[derive(Serialize, Deserialize)] - #[serde(deny_unknown_fields)] - struct Entry { - column: ColumnIdentity, - constraint: Constraint, - } - - pub(super) fn serialize( - columns: &BTreeMap, - serializer: S, - ) -> Result { - let entries: Vec = columns - .iter() - .map(|(column, constraint)| Entry { - column: column.clone(), - constraint: constraint.clone(), - }) - .collect(); - entries.serialize(serializer) - } - - pub(super) fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - let entries = Vec::::deserialize(deserializer)?; - Ok(entries - .into_iter() - .map(|entry| (entry.column, entry.constraint)) - .collect()) - } -} - -/// A `Bound` in JSON: `{"included": v}`, `{"excluded": v}` or `"unbounded"`. -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -enum WireBound { - Included(T), - Excluded(T), - Unbounded, -} - -impl From<&Bound> for WireBound { - fn from(bound: &Bound) -> Self { - match bound { - Bound::Included(v) => Self::Included(v.clone()), - Bound::Excluded(v) => Self::Excluded(v.clone()), - Bound::Unbounded => Self::Unbounded, - } - } -} - -impl From> for Bound { - fn from(bound: WireBound) -> Self { - match bound { - WireBound::Included(v) => Bound::Included(v), - WireBound::Excluded(v) => Bound::Excluded(v), - WireBound::Unbounded => Bound::Unbounded, - } - } -} - -/// A column of the definition child's output, by `(table, name)`. A -/// `Project` below renames its columns and sets their table to its -/// qualifier (none by default). -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ColumnIdentity { - pub(crate) table: Option, - pub(crate) name: String, -} - -#[cfg(any(test, feature = "test-support"))] -impl ColumnIdentity { - pub fn new(table: Option, name: impl Into) -> Self { - Self { - table, - name: name.into(), - } - } -} - -/// A constraint on one column of a selection. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Constraint { - /// The value is one of these. - In(Vec), - /// The value is none of these. - NotIn(Vec), - /// The value lies between `lower` and `upper`; each end is `Included`, - /// `Excluded` or `Unbounded`. A PromQL range `m[r]` evaluated at t reads - /// the samples in (t − r, t]: left-open, right-closed. Examples, on the - /// timestamp column `ts` (ms) unless stated: - /// - /// | Query fragment | `lower` | `upper` | - /// |---|---|---| - /// | PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | - /// | PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | - /// | PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | - /// | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | - /// | SQL `WHERE price > 10` (column `price`) | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | - Interval { - #[serde(with = "limit_bound")] - lower: Bound, - #[serde(with = "limit_bound")] - upper: Bound, - }, -} - -/// An end of a [`Constraint::Interval`]. Two limits compare only when they -/// are of the same variant: a fixed value against a fixed value of the same -/// type, an offset from the evaluation time against another such offset. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum IntervalConstraintLimit { - /// A constant: `Float64(10.0)` for SQL `price > 10`; - /// `Timestamp(1_700_000_000_000)` for PromQL `@ 1700000000` or a SQL - /// timestamp literal. - FixedValue(ScalarValue), - /// The evaluation time t of a repeatedly evaluated query plus this many - /// ms (negative: before t); only on the timestamp column. `m[5m]` is - /// `(EvaluationTimePlusMs(-300_000), EvaluationTimePlusMs(0)]`. - EvaluationTimePlusMs(i64), -} - -/// A `Constraint::Interval` bound as a [`WireBound`]. -mod limit_bound { - use super::{IntervalConstraintLimit, WireBound}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use std::ops::Bound; - - pub(super) fn serialize( - bound: &Bound, - serializer: S, - ) -> Result { - WireBound::from(bound).serialize(serializer) - } - - pub(super) fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - Ok(WireBound::deserialize(deserializer)?.into()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum CoverageError { - /// The node is not a summary state (`SummaryAgg`, `SummaryTumblingWindowChain` or - /// `SummaryMerge`): a caller bug, or bad input when it is a merge's - /// input. - #[error( - "coverage is derived only for SummaryAgg, SummaryTumblingWindowChain and SummaryMerge" - )] - NotSummary, - /// A `SummaryMerge` without inputs: bad input. - #[error("summary merge requires at least one input")] - EmptyMerge, - /// The inputs of a `SummaryMerge` compute different things: bad input. - #[error("summary merge inputs compute different things")] - DefinitionMismatch, - /// The inputs of a `SummaryMerge` are not proven to relate as the - /// family's merge requires (disjoint, or disjoint in time for counter - /// states): bad input. - #[error( - "summary merge inputs are not proven {}, which the family's merge requires", - match required { - MergeInputRule::DisjointInTime => "disjoint in time", - _ => "disjoint", - } - )] - InputsOverlap { required: MergeInputRule }, - /// A regrouping `SummaryMerge` key that is not one of the group keys - /// its input states are built by (a build grouped `without` keys has - /// none): bad input. - #[error("regrouping merge key {key} is not a group key of its input states")] - NotAGroupKey { key: ColumnId }, - /// A regrouping `SummaryMerge` of states whose summary type does not - /// merge different groups: counter states (`Rate`, `Increase`) merge - /// consecutive windows of one series, a summary type without a merge - /// merges nothing, and a Hydra's groups share one state. Bad input. - #[error("the inputs' summary type cannot merge different groups")] - SummaryTypeCannotRegroup, - /// The build a regrouping `SummaryMerge` stands for does not derive: - /// bad input. - #[error("the regrouped build does not derive: {reason}")] - RegroupedBuild { reason: String }, -} - -/// A memo group read as an input, and the inputs a rule builds over such -/// groups. -type GroupInput<'m> = InputOrTemporaryBuild>; - -impl SummaryCoverage { - /// Coverage of a summary state a memo group computes with `operator` - /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, - /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped - /// `SummaryAgg`. A merge reads its inputs' stored coverage and fails - /// unless every input has the same definition and their selections are - /// pairwise disjoint; a family whose merge is idempotent - /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may - /// also merge overlapping selections, and a counter family - /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. - pub(crate) fn derive_from_inputs<'m>( - operator: &Operator>, - ) -> Result { - let as_built = |op: &ASAPOp>| -> ASAPOp> { - op.map_children(|input| InputOrTemporaryBuild::Input(*input)) - }; - match operator { - Operator::NonASAP(_) => match selected_build(operator)? { - Some(build) => of_summary_agg(&build), - None => Err(CoverageError::NotSummary), - }, - Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( - "#1207 PR 2: a chain's coverage is a build's over its input, with the timestamp column unconstrained" - ), - Operator::ASAP(ASAPOp::SummaryMerge { - children, group_by, .. - }) => { - let inputs = children - .iter() - .map(|child| { - child - .aggregate_coverage() - .cloned() - .ok_or(CoverageError::NotSummary) - }) - .collect::, _>>()?; - let merged = of_merge(&inputs)?; - match group_by { - None => Ok(merged), - Some(keys) => regrouped(merged, &children[0], keys), - } - } - Operator::ASAP(_) => Err(CoverageError::NotSummary), - } - } - - /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, - /// of its input group and unlifted operators. Two equal definitions - /// (the equality a merge requires) of one memo get equal ids, so a - /// summary store can key states by this id and a concrete selection. - pub(crate) fn definition_id(&self) -> String { - const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; - const PRIME: u128 = 0x0000000001000000000000000000013b; - let mut hash = OFFSET; - for byte in format!("{:?}", self.definition).bytes() { - hash ^= u128::from(byte); - hash = hash.wrapping_mul(PRIME); - } - format!("{hash:032x}") - } -} - -/// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by -/// their key columns, the build that computes exactly the kept states: the -/// same build over its input rows filtered by the same predicate on the -/// input columns those keys name. `None` for any other operator. -fn selected_build<'m>( - operator: &Operator>, -) -> Result>>, CoverageError> { - let Operator::NonASAP(NonASAPOp::Filter { - pred, - child: states, - }) = operator - else { - return Ok(None); - }; - if operator.output_kind() != OperatorResultKind::State { - return Ok(None); - } - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction: Reduction::Reduce(by), - filter, - }) = states.operator() - else { - return Err(CoverageError::NotSummary); - }; - if by.is_without() { - return Err(CoverageError::NotSummary); - } - // Each state key column is the input column of the same name it groups by. - let mut unmapped = None; - let rows_pred = pred.0.map_columns(&mut |column| { - let mapped = by.keys().get(column).copied().filter(|&input_column| { - matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), - (Some(at_state), Some(at_input)) if at_state.name == at_input.name) - }); - mapped.unwrap_or_else(|| { - unmapped = Some(column); - column - }) - }); - if let Some(key) = unmapped { - return Err(CoverageError::NotAGroupKey { key }); - } - let mut built = |input: &MemoGroupAsInput<'m>| InputOrTemporaryBuild::Input(*input); - let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { - reason: error.to_string(), - }; - let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { - pred: Predicate(rows_pred.map_operator_refs(&mut built)), - child: InputOrTemporaryBuild::Input(child), - }) - .map_err(regrouped_build)?; - let reduction = Reduction::Reduce(by); - summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; - Ok(Some(ASAPOp::SummaryAgg { - child: rows, - summary_type: family, - input, - reduction, - filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), - })) -} - -/// The coverage of a merge of states covering `inputs`. -fn of_merge(inputs: &[SummaryCoverage]) -> Result { - let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; - if inputs - .iter() - .any(|input| input.definition != first.definition) - { - return Err(CoverageError::DefinitionMismatch); - } - // A family without a merge is checked as the strictest rule; the - // executor rejects the merge itself. - let required = match first.definition.unlifted_operators.first() { - Some(Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - .. - })) => SummaryTypeCapability::of(family).merge_inputs, - _ => None, - } - .unwrap_or(MergeInputRule::Disjoint); - let separated: fn(&SelectionBox, &SelectionBox) -> bool = match required { - MergeInputRule::MayOverlap => |_, _| true, - MergeInputRule::DisjointInTime => todo!( - "#1207 PR 2: disjoint intervals on the timestamp column, with IntervalConstraintLimit ends" - ), - MergeInputRule::Disjoint => SelectionBox::disjoint, - }; - // Every pair of boxes of different inputs. - if required != MergeInputRule::MayOverlap - && inputs.iter().enumerate().any(|(i, input)| { - inputs[..i].iter().any(|other| { - input - .selection - .iter() - .any(|a| other.selection.iter().any(|b| !separated(a, b))) - }) - }) - { - return Err(CoverageError::InputsOverlap { required }); - } - Ok(SummaryCoverage { - definition: first.definition.clone(), - selection: union(inputs.iter().flat_map(|c| c.selection.clone()).collect()), - }) -} - -/// The coverage of a regrouping merge, keyed by `keys`, of states whose -/// merge covers `merged` (`first` is its first input): the same rows -/// (`selection`), computed by the input's build grouped by the columns -/// `keys` name. The input states of one output group are different groups -/// of one build, so their rows are disjoint. A counter family merges only -/// time-disjoint windows, never different groups at one time. -fn regrouped( - merged: SummaryCoverage, - first: &MemoGroupAsInput<'_>, - keys: &[ColumnId], -) -> Result { - let Some(Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - input, - reduction, - filter, - .. - })) = merged.definition.unlifted_operators.first() - else { - return Err(CoverageError::NotSummary); - }; - if !SummaryTypeCapability::merges_groups(family) { - return Err(CoverageError::SummaryTypeCannotRegroup); - } - let state = first.schema(); - let rows = build_input_schema(first).ok_or(CoverageError::NotSummary)?; - // The build's input column each key of the state names. - let mapped = keys - .iter() - .map(|&key| { - let column = match reduction { - Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), - // A per-series state keeps its input's columns in place. - Reduction::PerEntity => Some(key), - Reduction::Reduce(_) => None, - }; - column - .filter(|&column| { - let (Some(at_state), Some(at_input)) = - (state.fields.get(key), rows.fields.get(column)) - else { - return false; - }; - at_state.is_plain() - && at_state.name == at_input.name - && at_state.table == at_input.table - }) - .ok_or(CoverageError::NotAGroupKey { key }) - }) - .collect::, _>>()?; - let reduction = Reduction::by(mapped); - summary_agg_schema(&rows, family, &reduction).map_err(|error| { - CoverageError::RegroupedBuild { - reason: error.to_string(), - } - })?; - // A direct build's definition, so its definition id is the same. - let mut definition = merged.definition.clone(); - definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction, - filter: filter.clone(), - }); - Ok(SummaryCoverage { - definition, - selection: merged.selection, - }) -} - -/// The schema of the rows the build of summary state `state` reads (its -/// `SummaryAgg`'s input; one build over a chain's window reads its range of -/// the chain's input, which has the input's schema). `None` when `state` -/// is no summary state. -fn build_input_schema(state: &I) -> Option { - if let Some(states) = selected_from(state) { - return build_input_schema(&states); - } - match state.operator() { - Operator::ASAP( - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => Some(child.schema().clone()), - Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { - build_input_schema(children.first()?) - } - _ => None, - } -} - -/// The coverage of `build`, a `SummaryAgg` over memo groups (or over an -/// input a rule built over them): its operators walked down through the -/// liftable operators of its input groups' expressions. -fn of_summary_agg(build: &ASAPOp>) -> Result { - let ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction, - filter, - } = build - else { - unreachable!("called on a SummaryAgg"); - }; - // The operators a predicate can be lifted through, top to bottom, and - // the first input below them. A subquery's group stays named in the - // residual of the operator that reads it. - let mut chain: Vec>> = Vec::new(); - let mut base = child.clone(); - loop { - let operator = base.operator(); - let passes = match &operator { - Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, - Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), - _ => false, - }; - if !passes { - break; - } - // The main input is an operator's first child. - base = operator.children()[0].clone(); - chain.push(operator); - } - let InputOrTemporaryBuild::Input(input_group) = &base else { - unreachable!("a rule builds only liftable operators over memo groups"); - }; - let top = child.schema(); - // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual - // of `chain[d]` when it is a `Filter`. - let mut lifted = BTreeMap::new(); - let agg_filter = filter - .as_ref() - .and_then(|pred| residual(pred, top, &[], &mut lifted)); - let kept: Vec>>> = chain - .iter() - .enumerate() - .map(|(depth, link)| match link { - Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { - residual(pred, top, &chain[..depth], &mut lifted) - } - _ => None, - }) - .collect(); - // A `Scan`'s predicates (PromQL label matchers) are lifted too; when - // one is, the `Scan` with the rest of its predicates is the last - // unlifted operator, and there is no input group. - let scan = match base.operator() { - Operator::NonASAP(NonASAPOp::Scan { - source, - predicates, - schema, - }) => { - let lifted_above = lifted.clone(); - let kept = predicates - .iter() - .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) - .collect(); - (lifted != lifted_above).then_some(NonASAPOp::Scan { - source, - predicates: kept, - schema, - }) - } - _ => None, - }; - - let mut unlifted_operators = vec![Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - filter: agg_filter, - })]; - for (depth, link) in chain.iter().enumerate() { - match link { - Operator::NonASAP(NonASAPOp::Filter { .. }) => { - if let Some(rest) = kept[depth].clone() { - unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { - pred: rest, - child: None, - })); - } - } - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) => todo!( - "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with IntervalConstraintLimit ends" - ), - other => unlifted_operators.push(unlifted(other)), - } - } - let input_group = match scan { - Some(scan) => { - unlifted_operators.push(Operator::NonASAP(scan)); - None - } - None => Some(input_group.group_id), - }; - - let selection = SelectionBox { - columns: lifted - .into_iter() - .map(|(column, constraint)| { - let field = &top.fields[column]; - let identity = ColumnIdentity { - table: field.table.clone(), - name: field.name.clone(), - }; - (identity, constraint) - }) - .collect(), - }; - Ok(SummaryCoverage { - definition: SummaryStateDefinition { - input_group, - unlifted_operators, - }, - selection: vec![selection], - }) -} - -/// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest, each scalar subquery -/// naming the group it reads. `above` -/// are the operators between the predicate and the `SummaryAgg`, top to -/// bottom. A column whose `(table, name)` is not unique in `top` cannot be -/// named in a selection, so its conjuncts stay. -fn residual( - pred: &Predicate>, - top: &Schema, - above: &[Operator>], - lifted: &mut BTreeMap, -) -> Option>> { - let mut rest: Vec>> = Vec::new(); - for conjunct in pred.0.conjuncts() { - let lift = constraint_of(conjunct).and_then(|(column, constraint)| { - let column = column_at_top(column, above)?; - let field = &top.fields[column]; - let namesakes = top - .fields - .iter() - .filter(|f| f.name == field.name && f.table == field.table) - .count(); - if namesakes != 1 { - return None; - } - let combined = match lifted.get(&column) { - Some(existing) => existing.intersect(&constraint)?, - None => constraint, - }; - Some((column, combined)) - }); - match lift { - Some((column, constraint)) => { - lifted.insert(column, constraint); - } - None => rest.push(conjunct.map_operator_refs(&mut group_id)), - } - } - match rest.len() { - 0 => None, - 1 => rest.pop().map(Predicate), - _ => Some(Predicate(ScalarExpr::BoolAnd(rest))), - } -} - -/// The memo group `input` is; `None` for a rule's temporary build. -fn group_id(input: &GroupInput<'_>) -> Option { - match input { - InputOrTemporaryBuild::Input(group) => Some(group.group_id), - InputOrTemporaryBuild::TemporaryBuild(_) => None, - } -} - -/// `link`, a liftable operator kept whole in a definition: its main input -/// `None`, every other input the group it is. -fn unlifted(link: &Operator>) -> Operator> { - let mut operator = link.map_children(group_id); - match &mut operator { - Operator::NonASAP( - NonASAPOp::Filter { child, .. } - | NonASAPOp::Project { child, .. } - | NonASAPOp::TimeRange { child, .. } - | NonASAPOp::TimeShift { child, .. }, - ) => *child = None, - _ => unreachable!("only liftable operators are walked"), - } - operator -} - -/// Map a column of the input of `above.last()` up to the agg child's output: -/// a `Project` passes it only as a direct column item. -fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { - for link in above.iter().rev() { - if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { - column = cols - .iter() - .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; - } - } - Some(column) -} - -/// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of -/// equalities on one column, with non-null literals. -fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { - let literal = |e: &ScalarExpr| match e { - ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), - _ => None, - }; - match expr { - ScalarExpr::Compare { - left, op, right, .. - } => { - let (column, value, op) = match (left.as_ref(), right.as_ref()) { - (ScalarExpr::Column(c), r) => (*c, literal(r)?, op.clone()), - (l, ScalarExpr::Column(c)) => (*c, literal(l)?, flip(op)?), - _ => return None, - }; - let constraint = match op { - CompareOpKind::Eq => Constraint::In(vec![value]), - CompareOpKind::Ne => Constraint::NotIn(vec![value]), - CompareOpKind::Lt => interval(Bound::Unbounded, Bound::Excluded(value)), - CompareOpKind::Le => interval(Bound::Unbounded, Bound::Included(value)), - CompareOpKind::Gt => interval(Bound::Excluded(value), Bound::Unbounded), - CompareOpKind::Ge => interval(Bound::Included(value), Bound::Unbounded), - _ => return None, - }; - Some((column, constraint)) - } - ScalarExpr::InList { - expr, - list, - negated, - } => { - let ScalarExpr::Column(column) = expr.as_ref() else { - return None; - }; - let values = list.iter().map(literal).collect::>>()?; - let values = dedup(values); - Some(( - *column, - if *negated { - Constraint::NotIn(values) - } else { - Constraint::In(values) - }, - )) - } - ScalarExpr::BoolOr(disjuncts) => { - let mut column = None; - let mut values = Vec::new(); - for disjunct in disjuncts { - let (c, Constraint::In(v)) = constraint_of(disjunct)? else { - return None; - }; - if column.replace(c).is_some_and(|prev| prev != c) { - return None; - } - values.extend(v); - } - Some((column?, Constraint::In(dedup(values)))) - } - _ => None, - } -} - -fn flip(op: &CompareOpKind) -> Option { - Some(match op { - CompareOpKind::Eq => CompareOpKind::Eq, - CompareOpKind::Ne => CompareOpKind::Ne, - CompareOpKind::Lt => CompareOpKind::Gt, - CompareOpKind::Le => CompareOpKind::Ge, - CompareOpKind::Gt => CompareOpKind::Lt, - CompareOpKind::Ge => CompareOpKind::Le, - _ => return None, - }) -} - -/// An interval between two fixed values. -fn interval(lower: Bound, upper: Bound) -> Constraint { - Constraint::Interval { - lower: lower.map(IntervalConstraintLimit::FixedValue), - upper: upper.map(IntervalConstraintLimit::FixedValue), - } -} - -fn dedup(values: Vec) -> Vec { - let mut out: Vec = Vec::new(); - for v in values { - if !out.contains(&v) { - out.push(v); - } - } - out -} - -/// Whether `v` equals one of `values`; `None` when some pair cannot be -/// compared (different types, NaN), since `1` and `1.0` may select the same -/// rows. -fn among(v: &ScalarValue, values: &[ScalarValue]) -> Option { - for w in values { - if compare(v, w)? == Ordering::Equal { - return Some(true); - } - } - Some(false) -} - -/// The values of `a` that are (`keep = true`) or are not in `b`. -fn filter_values(a: &[ScalarValue], b: &[ScalarValue], keep: bool) -> Option> { - let mut out = Vec::new(); - for v in a { - if among(v, b)? == keep { - out.push(v.clone()); - } - } - Some(out) -} - -/// Order of two literals of the same type; `None` across types or for NaN. -fn compare(a: &ScalarValue, b: &ScalarValue) -> Option { - match (a, b) { - (ScalarValue::Int64(a), ScalarValue::Int64(b)) => Some(a.cmp(b)), - (ScalarValue::Float64(a), ScalarValue::Float64(b)) => a.partial_cmp(b), - (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => Some(a.cmp(b)), - (ScalarValue::Boolean(a), ScalarValue::Boolean(b)) => Some(a.cmp(b)), - (ScalarValue::Timestamp(a), ScalarValue::Timestamp(b)) => Some(a.cmp(b)), - _ => None, - } -} - -/// Order of two interval limits; `None` across variants or when the fixed -/// values do not compare. -fn compare_limits(a: &IntervalConstraintLimit, b: &IntervalConstraintLimit) -> Option { - use IntervalConstraintLimit::*; - match (a, b) { - (FixedValue(a), FixedValue(b)) => compare(a, b), - (EvaluationTimePlusMs(a), EvaluationTimePlusMs(b)) => Some(a.cmp(b)), - _ => None, - } -} - -/// Whether value `v` lies inside the interval; `None` when unsure. -fn value_inside( - v: &ScalarValue, - bounds: ( - &Bound, - &Bound, - ), -) -> Option { - inside( - &IntervalConstraintLimit::FixedValue(v.clone()), - bounds, - compare_limits, - ) -} - -/// Whether `v` lies inside the interval; `None` when it cannot be compared. -fn inside( - v: &T, - (lower, upper): (&Bound, &Bound), - cmp: impl Fn(&T, &T) -> Option, -) -> Option { - let above = match lower { - Bound::Unbounded => true, - Bound::Included(l) => cmp(v, l)? != Ordering::Less, - Bound::Excluded(l) => cmp(v, l)? == Ordering::Greater, - }; - let below = match upper { - Bound::Unbounded => true, - Bound::Included(u) => cmp(v, u)? != Ordering::Greater, - Bound::Excluded(u) => cmp(v, u)? == Ordering::Less, - }; - Some(above && below) -} - -/// Whether no value is both at most `upper` and at least `lower`. -fn ends_before( - upper: &Bound, - lower: &Bound, - cmp: impl Fn(&T, &T) -> Option, -) -> bool { - match (upper, lower) { - (Bound::Included(u), Bound::Included(l)) => cmp(u, l) == Some(Ordering::Less), - (Bound::Included(u) | Bound::Excluded(u), Bound::Included(l) | Bound::Excluded(l)) => { - matches!(cmp(u, l), Some(Ordering::Less | Ordering::Equal)) - } - _ => false, - } -} - -fn intervals_disjoint( - (al, au): (&Bound, &Bound), - (bl, bu): (&Bound, &Bound), - cmp: impl Fn(&T, &T) -> Option + Copy, -) -> bool { - ends_before(au, bl, cmp) || ends_before(bu, al, cmp) -} - -/// The tighter of two lower (`want = Greater`) or upper (`Less`) bounds. -fn tighter( - a: &Bound, - b: &Bound, - want: Ordering, -) -> Option> { - let value = |bound: &Bound| match bound { - Bound::Included(v) | Bound::Excluded(v) => Some(v.clone()), - Bound::Unbounded => None, - }; - Some(match (value(a), value(b)) { - (None, _) => b.clone(), - (_, None) => a.clone(), - (Some(x), Some(y)) => match compare_limits(&x, &y)? { - Ordering::Equal if matches!(a, Bound::Excluded(_)) => a.clone(), - Ordering::Equal => b.clone(), - order if order == want => a.clone(), - _ => b.clone(), - }, - }) -} - -impl Constraint { - /// The conjunction of two constraints on one column, when it is again a - /// single constraint. - fn intersect(&self, other: &Self) -> Option { - use Constraint::*; - Some(match (self, other) { - (In(a), In(b)) => In(filter_values(a, b, true)?), - (In(a), NotIn(b)) | (NotIn(b), In(a)) => In(filter_values(a, b, false)?), - (NotIn(a), NotIn(b)) => NotIn(dedup(a.iter().chain(b).cloned().collect())), - (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => { - let mut kept = Vec::new(); - for v in a { - if value_inside(v, (lower, upper))? { - kept.push(v.clone()); - } - } - In(kept) - } - ( - Interval { - lower: al, - upper: au, - }, - Interval { - lower: bl, - upper: bu, - }, - ) => Interval { - lower: tighter(al, bl, Ordering::Greater)?, - upper: tighter(au, bu, Ordering::Less)?, - }, - (NotIn(_), Interval { .. }) | (Interval { .. }, NotIn(_)) => return None, - }) - } - - /// Whether no value satisfies both; `false` when unsure. - fn disjoint(&self, other: &Self) -> bool { - use Constraint::*; - match (self, other) { - (In(a), In(b)) => a.iter().all(|v| among(v, b) == Some(false)), - (In(a), NotIn(b)) | (NotIn(b), In(a)) => a.iter().all(|v| among(v, b) == Some(true)), - (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => a - .iter() - .all(|v| value_inside(v, (lower, upper)) == Some(false)), - ( - Interval { - lower: al, - upper: au, - }, - Interval { - lower: bl, - upper: bu, - }, - ) => intervals_disjoint((al, au), (bl, bu), compare_limits), - (NotIn(_), _) | (_, NotIn(_)) => false, - } - } -} - -#[cfg(any(test, feature = "test-support"))] -impl SelectionBox { - pub fn new(columns: BTreeMap) -> Self { - Self { columns } - } - - /// The per-column constraints. - pub fn columns(&self) -> &BTreeMap { - &self.columns - } -} - -impl SelectionBox { - /// Whether no row lies in both boxes: some column is restricted by both - /// and the restrictions are disjoint. - fn disjoint(&self, other: &Self) -> bool { - self.columns - .iter() - .any(|(column, a)| other.columns.get(column).is_some_and(|b| a.disjoint(b))) - } -} - -/// Join boxes that differ only in one column whose union is again one -/// constraint: touching value ranges, or the values of one `In` column. -/// Gaps stay separate boxes. -fn union(mut boxes: Vec) -> Vec { - let mut i = 0; - while i < boxes.len() { - let joined = (i + 1..boxes.len()).find_map(|j| join(&boxes[i], &boxes[j]).map(|b| (j, b))); - match joined { - Some((j, joined)) => { - boxes.remove(j); - boxes[i] = joined; - i = 0; - } - None => i += 1, - } - } - boxes -} - -fn join(a: &SelectionBox, b: &SelectionBox) -> Option { - if a == b { - return Some(a.clone()); - } - if a.columns.len() != b.columns.len() { - return None; - } - let mut differing = a - .columns - .iter() - .filter(|(column, constraint)| b.columns.get(*column) != Some(*constraint)); - let (column, constraint) = differing.next()?; - if differing.next().is_some() { - return None; - } - let joined = match (constraint, b.columns.get(column)?) { - (Constraint::In(values), Constraint::In(more)) => { - Constraint::In(dedup(values.iter().chain(more).cloned().collect())) - } - ( - Constraint::Interval { - lower: al, - upper: au, - }, - Constraint::Interval { - lower: bl, - upper: bu, - }, - ) => { - let (lower, upper) = touching((al, au), (bl, bu), compare_limits)?; - Constraint::Interval { lower, upper } - } - _ => return None, - }; - let mut columns = a.columns.clone(); - columns.insert(column.clone(), joined); - Some(SelectionBox { columns }) -} - -/// The union of two intervals when one ends exactly where the other starts, -/// with the shared end point in exactly one of them. -fn touching( - (al, au): (&Bound, &Bound), - (bl, bu): (&Bound, &Bound), - cmp: impl Fn(&T, &T) -> Option, -) -> Option<(Bound, Bound)> { - let meets = |upper: &Bound, lower: &Bound| match (upper, lower) { - (Bound::Included(u), Bound::Excluded(l)) | (Bound::Excluded(u), Bound::Included(l)) => { - cmp(u, l) == Some(Ordering::Equal) - } - _ => false, - }; - if meets(au, bl) { - Some((al.clone(), bu.clone())) - } else if meets(bu, al) { - Some((bl.clone(), au.clone())) - } else { - None - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::scalar::ExprSemantics; - use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; - - fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { - Constraint::Interval { - lower: Bound::Excluded(EvaluationTimePlusMs(lower_ms)), - upper: Bound::Included(EvaluationTimePlusMs(upper_ms)), - } - } - - /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. - #[test] - fn a_value_comparison_is_an_interval_between_fixed_values() { - let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(0)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), - semantics: ExprSemantics::Sql, - }; - assert_eq!( - constraint_of(&price_gt_10), - Some(( - 0, - Constraint::Interval { - lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), - upper: Bound::Unbounded, - } - )) - ); - } - - /// `m[5m]` and `m[5m] offset 5m` are disjoint and touch; neither - /// compares with an `@ T` interval. - #[test] - fn limits_compare_only_within_one_variant() { - let last_5m = relative(-300_000, 0); - let offset_5m = relative(-600_000, -300_000); - assert!(last_5m.disjoint(&offset_5m)); - let at = Constraint::Interval { - lower: Bound::Excluded(FixedValue(ScalarValue::Timestamp(1_699_999_700_000))), - upper: Bound::Included(FixedValue(ScalarValue::Timestamp(1_700_000_000_000))), - }; - assert!(!last_5m.disjoint(&at)); - assert_eq!(last_5m.intersect(&at), None); - let column = ColumnIdentity::new(None, "ts"); - let one = |constraint: Constraint| SelectionBox { - columns: BTreeMap::from([(column.clone(), constraint)]), - }; - assert_eq!( - union(vec![one(last_5m), one(offset_5m)]), - vec![one(relative(-600_000, 0))] - ); - } - - #[test] - fn an_interval_round_trips_through_json() { - let constraint = relative(-3_900_000, -3_600_000); - let json = serde_json::to_value(&constraint).unwrap(); - assert_eq!( - json, - serde_json::json!({"interval": { - "lower": {"excluded": {"evaluation_time_plus_ms": -3_900_000}}, - "upper": {"included": {"evaluation_time_plus_ms": -3_600_000}}, - }}) - ); - assert_eq!( - serde_json::from_value::(json).unwrap(), - constraint - ); - } -} diff --git a/crates/types/src/ir/schema/aggregate_schema.rs b/crates/types/src/ir/schema/aggregate_schema.rs index 2c1113534..5fe7edd11 100644 --- a/crates/types/src/ir/schema/aggregate_schema.rs +++ b/crates/types/src/ir/schema/aggregate_schema.rs @@ -10,6 +10,22 @@ use crate::ir::operator::AggIntent; use crate::ir::scalar::ColumnRef; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; use crate::ir::SchemaDerivationError; + +/// Where one output column of an aggregate comes from: an input column it +/// keeps, a measure's value (with the input column the schema types it by, +/// the measure's own or the probe value column), or a label column the +/// measure adds (`count_values`' and `absent`'s). `sum by (job) (x)` → +/// `[InputColumn(job), Measure { index: 0, input_column: Some(value) }]`. The one layout the output schema +/// ([`aggregate_output_schema`]) and the group properties read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OutputColumnSource { + InputColumn(ColumnId), + Measure { + index: usize, + input_column: Option, + }, + AddedLabel, +} /// Output schema of a *per-series* window/range reduction (`rate`/`increase`, /// or an `*_over_time` reducer under a time `Window`). Such a reduction emits /// one value per series, so every label column of `input` is preserved and only @@ -18,7 +34,7 @@ use crate::ir::SchemaDerivationError; fn per_series_reduction_schema( input: &Schema, agg: &AggIntent, -) -> Result { +) -> Result<(Schema, Vec), SchemaDerivationError> { let vi = if let Some(index) = agg.input_cols().first() { *index } else { @@ -34,6 +50,13 @@ fn per_series_reduction_schema( ))); } let mut columns = input.fields.clone(); + let mut sources: Vec = (0..columns.len()) + .map(OutputColumnSource::InputColumn) + .collect(); + sources[vi] = OutputColumnSource::Measure { + index: 0, + input_column: Some(vi), + }; { let mut out = agg.output_column(&columns[vi]); out.name = "value".into(); @@ -44,14 +67,17 @@ fn per_series_reduction_schema( out.dtype = FieldDataType::Plain(DataType::Float64); columns[vi] = out; } - Ok(Schema { - fields: columns, - time_index: input.time_index, - unique_keys: input.unique_keys.clone(), - // Per-series reduction is label-preserving: it inherits its input's - // completeness (an open scan stays open; a closed one stays closed). - closed: input.closed, - }) + Ok(( + Schema { + fields: columns, + time_index: input.time_index, + unique_keys: input.unique_keys.clone(), + // Per-series reduction is label-preserving: it inherits its input's + // completeness (an open scan stays open; a closed one stays closed). + closed: input.closed, + }, + sources, + )) } /// The output schema of an `Aggregate { reduction, measures }` over `in_schema` — @@ -71,6 +97,17 @@ pub fn aggregate_output_schema( measures: &[AggIntent], output_names: &[String], ) -> Result { + Ok(aggregate_output(in_schema, reduction, measures, output_names)?.0) +} + +/// [`aggregate_output_schema`] and where each of its columns comes from +/// ([`OutputColumnSource`]), in one pass. +pub(crate) fn aggregate_output( + in_schema: &Schema, + reduction: &Reduction, + measures: &[AggIntent], + output_names: &[String], +) -> Result<(Schema, Vec), SchemaDerivationError> { let by = match reduction { Reduction::PerEntity => { debug_assert_eq!( @@ -93,6 +130,7 @@ pub fn aggregate_output_schema( } let mut out_cols: Vec = Vec::with_capacity(by.len() + measures.len()); + let mut sources: Vec = Vec::with_capacity(by.len() + measures.len()); for &id in by.keys() { let c = in_schema .fields @@ -102,6 +140,7 @@ pub fn aggregate_output_schema( in_schema.fields.len(), ))?; out_cols.push(c.clone()); + sources.push(OutputColumnSource::InputColumn(id)); } let value_col_idx = crate::ir::scalar::column_resolution::resolve_column_ref( &ColumnRef::SampleValue, @@ -126,12 +165,17 @@ pub fn aggregate_output_schema( if let AggIntent::CountValues { label } = intent { if !out_cols.iter().any(|c| c.name == *label) { out_cols.push(Field::plain(label.clone(), DataType::Utf8, false)); + sources.push(OutputColumnSource::AddedLabel); } let mut cnt = intent.output_column(&probe); if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { cnt.name = name.clone(); } out_cols.push(cnt); + sources.push(OutputColumnSource::Measure { + index: i, + input_column: value_col_idx, + }); continue; } // `absent(v)` emits its synthesized labels as `Utf8` columns before @@ -142,15 +186,13 @@ pub fn aggregate_output_schema( .keys() .map(|name| Field::plain(name.clone(), DataType::Utf8, false)), ); + sources.extend(labels.keys().map(|_| OutputColumnSource::AddedLabel)); } // Only the output *type* is read from here, so the leading column is // enough for the multi-column intents: `Cardinality` and `PearsonCorr` // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); + let input_column = measure_input_column(in_schema, intent); + let in_col = input_column.map_or(&probe, |id| &in_schema.fields[id]); let mut out = intent.output_column(in_col); // A global extremum emits NULL for an empty input, even if its input // column is non-nullable. Grouped extrema only emit existing groups. @@ -168,6 +210,10 @@ pub fn aggregate_output_schema( out.name = name.clone(); } out_cols.push(out); + sources.push(OutputColumnSource::Measure { + index: i, + input_column: input_column.or(value_col_idx), + }); } // `count_values` groups by (by-keys ∪ the synthesized value label), so the // by-keys alone are not a unique key — be conservative and claim none. @@ -179,15 +225,29 @@ pub fn aggregate_output_schema( } else { vec![(0..by.len()).collect()] }; - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys, - // A cross-series aggregate enumerates exactly `by ++ measures`, so its output - // is closed even over an open input — this is where an open schema - // freezes to closed. - closed: true, - }) + Ok(( + Schema { + fields: out_cols, + time_index: None, + unique_keys, + // A cross-series aggregate enumerates exactly `by ++ measures`, so its output + // is closed even over an open input — this is where an open schema + // freezes to closed. + closed: true, + }, + sources, + )) +} + +/// The input column a measure's output is typed by when it names one: its +/// leading input column (the multi-column intents' output type ignores the +/// rest). +fn measure_input_column(in_schema: &Schema, intent: &AggIntent) -> Option { + intent + .input_cols() + .first() + .copied() + .filter(|&id| id < in_schema.fields.len()) } /// Output schema of a `without(excluded)` aggregate: the kept labels (every @@ -201,7 +261,7 @@ fn without_output_schema( excluded: &[ColumnId], measures: &[AggIntent], output_names: &[String], -) -> Result { +) -> Result<(Schema, Vec), SchemaDerivationError> { for &id in excluded { if id >= in_schema.fields.len() { return Err(SchemaDerivationError::InvalidGroupByColumn( @@ -211,6 +271,7 @@ fn without_output_schema( } } let mut out_cols: Vec = Vec::new(); + let mut sources: Vec = Vec::new(); for (i, col) in in_schema.fields.iter().enumerate() { let is_time = in_schema.time_index == Some(i); let is_value = crate::ir::scalar::column_resolution::resolve_column_ref( @@ -221,6 +282,7 @@ fn without_output_schema( == Some(i); if !is_time && !is_value && !excluded.contains(&i) { out_cols.push(col.clone()); + sources.push(OutputColumnSource::InputColumn(i)); } } let probe = in_schema @@ -232,11 +294,8 @@ fn without_output_schema( // Only the output *type* is read from here, so the leading column is // enough for the multi-column intents: `Cardinality` and `PearsonCorr` // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); + let input_column = measure_input_column(in_schema, intent); + let in_col = input_column.map_or(&probe, |id| &in_schema.fields[id]); let mut out = intent.output_column(in_col); if let Some((arg, _)) = intent .arg_selector_columns(in_schema) @@ -249,21 +308,162 @@ fn without_output_schema( out.name = name.clone(); } out_cols.push(out); + sources.push(OutputColumnSource::Measure { + index: i, + input_column: input_column.or_else(|| in_schema.column_id("value")), + }); } - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys: Vec::new(), - // The kept label set is runtime-only, so — unlike `by` — this does not - // freeze the open schema to closed. - closed: false, - }) + Ok(( + Schema { + fields: out_cols, + time_index: None, + unique_keys: Vec::new(), + // The kept label set is runtime-only, so — unlike `by` — this does not + // freeze the open schema to closed. + closed: false, + }, + sources, + )) } #[cfg(test)] mod tests { use super::*; + /// `[ts, value, job, host]`, the time series schema the layout tests read. + fn series() -> Schema { + Schema::with_time_index( + vec![ + col("ts", DataType::Timestamp, false), + col("value", DataType::Float64, false), + col("job", DataType::Utf8, false), + col("host", DataType::Utf8, false), + ], + 0, + vec![], + ) + } + + /// Each output column's source, and one source per column of + /// `aggregate_output_schema`'s output. + fn columns(reduction: Reduction, measures: &[AggIntent]) -> Vec { + let (schema, sources) = aggregate_output(&series(), &reduction, measures, &[]).unwrap(); + assert_eq!(sources.len(), schema.fields.len()); + sources + } + + /// `sum by (job)`: the key, then the measure. + #[test] + fn a_by_aggregate_keeps_its_keys_then_its_measures() { + use OutputColumnSource::*; + assert_eq!( + columns(Reduction::by(vec![2]), &[AggIntent::Sum { col: None }]), + vec![ + InputColumn(2), + Measure { + index: 0, + input_column: Some(1) + } + ] + ); + } + + /// `sum without (host)`: every label but `host`, then the measure. + #[test] + fn a_without_aggregate_keeps_the_other_labels_then_its_measures() { + use OutputColumnSource::*; + assert_eq!( + columns( + Reduction::Reduce(GroupKeys::without(vec![3])), + &[AggIntent::Sum { col: None }] + ), + vec![ + InputColumn(2), + Measure { + index: 0, + input_column: Some(1) + } + ] + ); + } + + /// `count_values by (job) ("version", v)` adds the `version` label before + /// the count; `count_values without (host)` adds none, like its schema. + #[test] + fn count_values_adds_its_label_only_by_keys() { + use OutputColumnSource::*; + let count_values = AggIntent::CountValues { + label: "version".into(), + }; + assert_eq!( + columns(Reduction::by(vec![2]), std::slice::from_ref(&count_values)), + vec![ + InputColumn(2), + AddedLabel, + Measure { + index: 0, + input_column: Some(1) + } + ] + ); + assert_eq!( + columns( + Reduction::Reduce(GroupKeys::without(vec![3])), + &[count_values] + ), + vec![ + InputColumn(2), + Measure { + index: 0, + input_column: Some(1) + } + ] + ); + } + + /// `absent(m{job="a", host="b"})` adds one label per matcher before its + /// value. + #[test] + fn absent_adds_its_labels_before_its_value() { + use OutputColumnSource::*; + let absent = AggIntent::Absent { + labels: std::collections::BTreeMap::from([ + ("host".to_string(), "b".to_string()), + ("job".to_string(), "a".to_string()), + ]), + }; + assert_eq!( + columns(Reduction::by(vec![]), &[absent]), + vec![ + AddedLabel, + AddedLabel, + Measure { + index: 0, + input_column: Some(1) + } + ] + ); + } + + /// A per-entity reduction keeps every input column, the value replaced + /// by the measure. + #[test] + fn a_per_entity_reduction_replaces_the_value_by_the_measure() { + use OutputColumnSource::*; + assert_eq!( + columns(Reduction::PerEntity, &[AggIntent::Rate]), + vec![ + InputColumn(0), + Measure { + index: 0, + input_column: Some(1) + }, + InputColumn(2), + InputColumn(3) + ] + ); + } + fn col(name: &str, dtype: DataType, nullable: bool) -> Field { Field::plain(name, dtype, nullable) } diff --git a/crates/types/src/ir/schema/error.rs b/crates/types/src/ir/schema/error.rs index d4f3b05b5..e84a61b43 100644 --- a/crates/types/src/ir/schema/error.rs +++ b/crates/types/src/ir/schema/error.rs @@ -16,6 +16,6 @@ pub enum SchemaDerivationError { EmptyConcat, #[error("invalid per-series sample column: {0}")] InvalidSampleColumn(String), - #[error("invalid summary coverage: {0}")] - Coverage(#[from] crate::ir::properties::summary_coverage::CoverageError), + #[error("invalid aggregate coverage: {0}")] + Coverage(#[from] crate::ir::properties::CoverageError), } diff --git a/crates/types/src/ir/schema/series_identity.rs b/crates/types/src/ir/schema/series_identity.rs index 94e997390..61bc53e69 100644 --- a/crates/types/src/ir/schema/series_identity.rs +++ b/crates/types/src/ir/schema/series_identity.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (Stage 0 hash-consing, `cse` in `asap_stage0_memo`). + // node is legal to share (hash-consing, `Memo::find_or_add_group`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 13ffd59b2..3ad853e70 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,6 +1,7 @@ //! Ranked-row output contract shared by logical TopK and its implementations. +use crate::ir::operator::arguments::Source; use crate::ir::operator::input::OperatorInput; -use crate::ir::operator::{Reduction, Source}; +use crate::ir::operator::Reduction; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; use crate::ir::schema::{ColumnId, DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; @@ -85,7 +86,7 @@ pub(crate) fn ranked_row_projection( alias: Some(output_name.into()), expr: ScalarExpr::Column(value), }); - let promql = reads_only_time_series(child); + let promql = child.scanned_sources().iter().all(Source::is_time_series); for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item @@ -142,12 +143,3 @@ pub fn series_identity_item( SummaryInputExpr::Tuple(items) } } - -fn reads_only_time_series(node: &I) -> bool { - match node.operator() { - Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { - matches!(source, Source::TimeSeries { .. }) - } - operator => operator.children().into_iter().all(reads_only_time_series), - } -} diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index c33162e00..d805699f6 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -6,7 +6,8 @@ //! [`ir::operator`] (§1 operators and their argument types), //! [`ir::scalar`] (§2.2 scalar expressions), [`ir::schema`] (§2.1 the //! per-edge schema) and [`ir::properties`] (§2.3 execution timing and -//! summary coverage), and the flat DAG form ([`ir::flat`]). +//! the selection algebra aggregate coverage is stated in), and the flat +//! DAG form ([`ir::flat`]). //! - [`summary_type`] — each summary type's parameters and predefined //! parameter lists, the statistics read from it and its capabilities. //! - [`accuracy`] — accuracy targets, the error metrics they are stated in, @@ -33,6 +34,6 @@ pub mod plan; pub mod serde_f64; pub mod stages; pub mod summary_type; -#[cfg(feature = "test-support")] +#[cfg(any(test, feature = "test-support"))] pub mod test_support; pub mod workload; diff --git a/crates/types/src/plan/materialized_view.rs b/crates/types/src/plan/materialized_view.rs index dd8d51737..5915bf633 100644 --- a/crates/types/src/plan/materialized_view.rs +++ b/crates/types/src/plan/materialized_view.rs @@ -64,20 +64,19 @@ //! it once. use std::collections::{HashMap, HashSet}; -use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; use super::physical_asap_dag::{ PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, + ViewCoverage, }; use crate::ir::operator::arguments::Source; use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::node::Operator; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::ExecutionTiming; -use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, SummaryCoverage}; use crate::stages::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is @@ -89,7 +88,7 @@ pub struct MaterializedView { pub node_id: PhysicalASAPNodeId, /// Which run refreshes it. pub(crate) refresh: MaterializedViewRefresh, - /// What the stored state is: the node's summary coverage. + /// What the stored state is: the node's aggregate coverage. pub(crate) coverage: ViewCoverage, /// What its refresh reads. pub inputs: MaterializedViewInputs, @@ -119,36 +118,6 @@ impl MaterializedView { } } -/// A view's summary coverage -/// ([`SummaryCoverage`], `docs/design_docs/proposals/asap-primitive-schema.md` -/// §4.2): what it computes and which rows went in. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ViewCoverage { - /// `SummaryCoverage::definition_id`: equal on every node, plan and - /// process that computes the same thing, whatever its selection; a - /// rollup to a coarser grouping computes something else and has its - /// own. - pub(crate) definition_id: String, - /// The rows that went in: a union of boxes of value sets, value ranges - /// and time relative to the evaluation time. A merge's is the union of - /// its inputs'. - pub(crate) selection: Vec, -} - -#[cfg(any(test, feature = "test-support"))] -impl ViewCoverage { - /// `SummaryCoverage::definition_id`. - pub fn definition_id(&self) -> &str { - &self.definition_id - } - - /// The rows that went in. - pub fn selection(&self) -> &[SelectionBox] { - &self.selection - } -} - /// What a view's refresh reads, each bound as a source of the refresh. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -349,7 +318,7 @@ pub enum InstallableDagError { ViewReadsUnwindowedSource { node_id: PhysicalASAPNodeId }, /// Unsatisfiable request: view `node_id` is not a summary state with a /// coverage (`reason`), so a runtime cannot say what it stores. - #[error("materialized view {node_id} has no summary coverage: {reason}")] + #[error("materialized view {node_id} has no aggregate coverage: {reason}")] ViewWithoutCoverage { node_id: PhysicalASAPNodeId, reason: String, @@ -593,14 +562,7 @@ impl PhysicalASAPDAG { .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| node.id) .collect(); - // The IR of each node, which coverage is derived from. - let ir = if views.is_empty() { - HashMap::new() - } else { - self.operator_nodes()? - }; let mut derived = Vec::new(); - let mut coverage_by_node: HashMap = HashMap::new(); for &id in index.children_first.iter().filter(|id| views.contains(id)) { let node = index.nodes[&id]; let (inputs, scans) = index.inputs_of_view(id, &views)?; @@ -627,13 +589,12 @@ impl PhysicalASAPDAG { ); } } - let coverage = coverage_of(id, &inputs, &ir, &coverage_by_node).map_err(|error| { + let coverage = node.view_coverage.clone().ok_or_else(|| { InstallableDagError::ViewWithoutCoverage { node_id: id, - reason: error.to_string(), + reason: "the plan node carries no view_coverage".into(), } })?; - coverage_by_node.insert(id, coverage.clone()); let view = MaterializedView { node_id: id, refresh: if refreshed_at_ingestion { @@ -641,10 +602,7 @@ impl PhysicalASAPDAG { } else { MaterializedViewRefresh::OnFirstRead }, - coverage: ViewCoverage { - definition_id: coverage.definition_id(), - selection: coverage.selection, - }, + coverage, inputs, retention_ms: node.retention_ms.unwrap_or(0), }; @@ -923,18 +881,3 @@ impl<'a> ViewIndex<'a> { Ok((inputs, scans)) } } - -/// The coverage of view `id` (IR `ir`), refreshed from `inputs`: the -/// coverage its memo group stores, which the realized plan will carry. -/// Coverage is derived only in the memo, never on node trees. -fn coverage_of( - id: PhysicalASAPNodeId, - inputs: &MaterializedViewInputs, - ir: &HashMap>, - coverage_by_node: &HashMap, -) -> Result { - let _ = (id, inputs, ir, coverage_by_node); - todo!( - "#1207 PR 6: the view's coverage from its memo group's GroupProperties::aggregate_coverage" - ) -} diff --git a/crates/types/src/plan/physical_asap_dag.rs b/crates/types/src/plan/physical_asap_dag.rs index ebb2d33ed..730e144b5 100644 --- a/crates/types/src/plan/physical_asap_dag.rs +++ b/crates/types/src/plan/physical_asap_dag.rs @@ -9,6 +9,7 @@ //! (#509 Assumption 8): it does not change lifetime or timing. use std::collections::{HashMap, HashSet}; +#[cfg(any(test, feature = "test-support"))] use std::rc::Rc; use serde::{Deserialize, Serialize}; @@ -16,12 +17,15 @@ use thiserror::Error; use crate::ir::flat::NodeId; use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::node::Operator; +#[cfg(any(test, feature = "test-support"))] +use crate::ir::operator::node::OperatorNode; use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::scalar::ScalarExpr; use crate::accuracy::AccuracyGuarantee; use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; +use crate::ir::properties::selection::SelectionBox; use crate::ir::schema::Schema; use crate::stages::PhysicalCandidateId; use crate::stages::{Lifetime, PhysicalPropertiesError}; @@ -95,6 +99,50 @@ pub struct PhysicalASAPDAGNode { /// built by a custom pass. #[serde(skip)] pub selected_physical_candidate: Option, + /// For a materialized view (lifetime `AcrossEvaluations`), what it + /// stores: its memo group's aggregate coverage, carried into the plan + /// as a [`ViewCoverage`] (the selection, and the plan-scope content id + /// of the definition, [`ViewCoverage::definition_id`]). Set when the + /// plan is built from the memo, where the node's group is known; `None` + /// on every other node. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub view_coverage: Option, +} + +/// A view's aggregate coverage +/// ([`AggregateCoverage`](crate::ir::properties::AggregateCoverage), `docs/design_docs/proposals/asap-primitive-schema.md` +/// §4.2): what it computes and which rows went in. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ViewCoverage { + /// The plan-scope content id of what the view computes: its memo + /// coverage's `AggregateDefinition` with every input group expanded + /// recursively to its expression, down to the `Scan`s, holding no + /// memo group or plan node ids, canonically serialized and hashed + /// (`stages::view_coverage`). + /// Equal across plans and processes that compute the same thing, + /// whatever the selection, and different when they do not (a rollup to + /// a coarser grouping computes something else and has its own). Inside + /// one memo, coverages compare by `AggregateDefinition` `==` instead + /// and need no id. + pub(crate) definition_id: String, + /// The rows that went in: a union of boxes of value sets, value ranges + /// and time relative to the evaluation time. A merge's is the union of + /// its inputs'. + pub(crate) selection: Vec, +} + +#[cfg(any(test, feature = "test-support"))] +impl ViewCoverage { + /// The plan-scope content id of what the view computes. + pub fn definition_id(&self) -> &str { + &self.definition_id + } + + /// The rows that went in. + pub fn selection(&self) -> &[SelectionBox] { + &self.selection + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -256,11 +304,11 @@ impl PhysicalASAPDAG { } /// Every node of this DAG as IR, by node id, with its exported - /// guarantee (an operator node has no physical properties): what a caller derives - /// IR properties of a plan file's node from, such as its - /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). - /// Shared inputs stay shared. Validates the DAG first. - pub(crate) fn operator_nodes( + /// guarantee (an operator node has no physical properties): what + /// [`Self::query_roots`] reads. Shared inputs stay shared. Validates the + /// DAG first. + #[cfg(any(test, feature = "test-support"))] + fn operator_nodes( &self, ) -> Result>, PhysicalASAPDAGValidationError> { self.validate()?; @@ -607,6 +655,7 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( guarantee: flat_node.guarantee, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); } let dag = PhysicalASAPDAG { diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs new file mode 100644 index 000000000..a8a80d550 --- /dev/null +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -0,0 +1,59 @@ +//! Memo scope of aggregate coverage: the IR derives it +//! ([`crate::ir::properties::aggregate_coverage`]) over memo groups +//! (`R` = [`MemoGroupId`]) and the memo stores it per group +//! (`GroupProperties::aggregate_coverage`); here the plan's view of it, a +//! content id that names no group ([`view_coverage`]). + +use serde::Serialize; + +use crate::ir::Operator; +use crate::plan::physical_asap_dag::ViewCoverage; +use crate::stages::{Memo, MemoGroupId}; + +/// The plan-scope coverage of `group`: the content id of its expanded +/// definition and its selection; `None` for a group without aggregate +/// coverage. The definition's input groups are expanded to their +/// expressions recursively, down to the `Scan`s, so the id names no memo +/// group: two memos (or processes) that compute the same thing give the +/// same id, and different computations different ids. It is the 128-bit +/// FNV-1a hash, in hex, of the expanded definition's JSON (non-finite +/// floats serialized by `serde_f64`). Stage 3 sets it on a materialized +/// view's plan node (`PhysicalASAPDAGNode::view_coverage`). +pub fn view_coverage(memo: &Memo, group: MemoGroupId) -> Option { + let coverage = memo.groups()[group.0 as usize] + .group_properties + .aggregate_coverage + .clone()?; + let expanded = coverage.map_references(|input| expanded(memo, input)); + let serialized = + serde_json::to_string(&expanded.definition).expect("an expanded definition serializes"); + Some(ViewCoverage { + definition_id: fnv1a_128_hex(&serialized), + selection: expanded.selection, + }) +} + +/// What a memo group computes with no ids: its expression over its input +/// groups' expressions, expanded the same way down to the `Scan`s. +#[derive(Serialize)] +#[serde(transparent)] +struct ExpandedExpression(Box>); + +fn expanded(memo: &Memo, group: MemoGroupId) -> ExpandedExpression { + ExpandedExpression(Box::new( + memo.groups()[group.0 as usize] + .expression + .map_children(|&input| expanded(memo, input)), + )) +} + +/// The 128-bit FNV-1a hash of `text`, in hex: stable across processes and +/// Rust versions. +fn fnv1a_128_hex(text: &str) -> String { + const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; + const PRIME: u128 = 0x0000000001000000000000000000013b; + let hash = text.bytes().fold(OFFSET, |hash, byte| { + (hash ^ u128::from(byte)).wrapping_mul(PRIME) + }); + format!("{hash:032x}") +} diff --git a/crates/types/src/stages/logical/candidate.rs b/crates/types/src/stages/logical/candidate.rs index 618c228bc..6d62d4784 100644 --- a/crates/types/src/stages/logical/candidate.rs +++ b/crates/types/src/stages/logical/candidate.rs @@ -65,10 +65,11 @@ pub struct CandidateExpression { /// A candidate [`Memo::add_candidate`](crate::stages::Memo::add_candidate) rejects; the memo is unchanged. #[derive(Debug, thiserror::Error)] pub enum CandidateError { - /// The candidate's expression fails the node rules (schema derivation, - /// input validation, merge check): a bug in the rule that built it. - #[error("the candidate's expression breaks the node rules: {0}")] - Schema(SchemaDerivationError), + /// The candidate's expression is not valid over its input groups (its + /// output schema does not derive, an input fails validation, or a merge + /// reads states it may not merge): a bug in the rule that built it. + #[error("the candidate's expression is not valid over its input groups: {0}")] + NotValidOverInputGroups(SchemaDerivationError), /// Its output schema or output kind differs from the group's: it does /// not compute the group's result (a bug in the rule that built it). #[error("the candidate's output differs from group {group:?}'s")] @@ -84,7 +85,7 @@ pub enum CandidateError { pub struct LogicalProperties { /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for /// an exact operator. - parameters: Option, + pub(super) parameters: Option, } impl LogicalProperties { diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs new file mode 100644 index 000000000..a60c88cfe --- /dev/null +++ b/crates/types/src/stages/logical/column_properties.rs @@ -0,0 +1,514 @@ +//! Value sign: the sign of each output column of a memo group (concepts +//! doc, section 3, "Value sign"), derived once from the input groups' signs +//! and, for a `Scan`, the signs the data workload declares for its source's +//! columns. Error contracts that hold only for values of one sign read it. + +use std::collections::BTreeMap; + +use super::MemoGroupAsInput; +use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction}; +use crate::ir::operator::asap::{summary_agg_layout, summary_build}; +use crate::ir::operator::input::OperatorInput; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::expr_ir::{ArithmeticOpKind, ScalarValue}; +use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; +use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; +use crate::ir::schema::{FieldDataType, Schema}; +use crate::ir::{ASAPOp, NonASAPOp, Operator}; +use crate::summary_type::{ + ExactKind, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, +}; +use crate::workload::bounds::source_name; +use crate::workload::{series_facts, PrometheusMetricType, SourceFacts}; + +/// The sign of one column's values, a group property derived from the +/// input groups like lineage (concepts doc, section 3, "Value sign"); a +/// `Scan`'s are declared per column in the data workload's source facts. +/// Error contracts that hold only for values of one sign read it. +/// +/// A sign is the IEEE 754 sign bit, so `-0.0` is `NonPositive` and `+0.0` +/// `NonNegative`; with it the rules are exact: a product or quotient has +/// the XOR of its operands' sign bits, a sum of two values with one sign +/// bit keeps it, a negation flips it and `abs` clears it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValueSign { + /// Every non-NaN value has its sign bit clear (`+0.0`, positive, + /// `+inf`). + NonNegative, + /// Every non-NaN value has its sign bit set (`-0.0`, negative, `-inf`). + NonPositive, + /// Neither is known. + Unknown, +} + +use ValueSign::{NonNegative, NonPositive, Unknown}; + +type Input<'m> = MemoGroupAsInput<'m>; + +/// The sign of each column of `output`, the output schema of `operator`, +/// from its input groups' `column_signs` and, for a `Scan`, `source_facts`. +pub(super) fn column_signs( + operator: &Operator>, + output: &Schema, + source_facts: &BTreeMap, +) -> Vec { + let signs = match operator { + Operator::NonASAP(op) => non_asap_signs(op, output, source_facts), + Operator::ASAP(op) => asap_signs(op, output), + }; + debug_assert_eq!( + signs.len(), + output.fields.len(), + "one sign per output column of {}", + operator.kind_name() + ); + signs +} + +fn signs_of(input: &Input<'_>) -> Vec { + input.group_properties().column_signs.clone() +} + +fn non_asap_signs( + op: &NonASAPOp>, + output: &Schema, + source_facts: &BTreeMap, +) -> Vec { + use NonASAPOp::*; + let unknown = || vec![Unknown; output.fields.len()]; + match op { + // The signs its source's facts declare (a histogram's or summary's + // series reads its family's); a counter's `value` is non-negative + // without a declaration. + Scan { source, schema, .. } => { + let facts = series_facts(source_facts, source_name(source)); + schema + .fields + .iter() + .map(|field| match facts { + Some((facts, series_type)) => facts + .column_signs + .get(&field.name) + .copied() + .or((field.name == "value" + && series_type == Some(PrometheusMetricType::Counter)) + .then_some(NonNegative)) + .unwrap_or(Unknown), + None => Unknown, + }) + .collect() + } + Values { .. } => unknown(), + // The scalar's value, every other column `Unknown`. + PromqlVectorFromScalar(expr) => { + let mut signs = unknown(); + if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, output) { + signs[value] = expression_sign(expr, &[]); + } + signs + } + Filter { child, .. } + | Sort { child, .. } + | Limit { child, .. } + | Dedup { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | PromqlSubquery { child, .. } + | PromqlSeriesSample { child, .. } + | PromqlInfoEnrich { child, .. } => signs_of(child), + // The relabeled column is written; every other column is copied. + PromqlRelabel { dst, child, .. } => { + let mut signs = signs_of(child); + match child.schema().column_id(dst) { + Some(written) => signs[written] = Unknown, + None => signs.push(Unknown), + } + signs + } + Project { cols, child, .. } => project_signs(cols, &signs_of(child)), + Aggregate { + reduction, + measures, + child, + .. + } => { + if let [AggIntent::TopK { .. }] = measures.as_slice() { + // The ranked rows, projected: each column copied. + return match crate::ir::schema::topk::ranked_row_projection(child, reduction, "") { + Ok(cols) => project_signs(&cols, &signs_of(child)), + Err(_) => unknown(), + }; + } + aggregate_signs(child, reduction, measures, output) + } + Join { + kind, left, right, .. + } => { + let mut signs = signs_of(left); + if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { + signs.extend(signs_of(right)); + } + signs + } + SetOp { left, right, .. } => agreed(&[signs_of(left), signs_of(right)]), + Concat { children } => agreed(&children.iter().map(signs_of).collect::>()), + SQLWindowFunc { child, .. } => { + let mut signs = signs_of(child); + signs.push(Unknown); + signs + } + // The left side's columns, its value per the operator, then the + // labels the match adds (unknown). + BinaryOp { + operator, lhs, rhs, .. + } => { + let mut signs = signs_of(lhs); + if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { + let rhs_value = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) + .ok() + .map_or(Unknown, |column| signs_of(rhs)[column]); + signs[value] = match &operator.kind { + // A filter keeps the left side's values. + BinaryOpKind::Compare(_) + | BinaryOpKind::Set( + PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless, + ) => signs[value], + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(signs[value], rhs_value), + BinaryOpKind::CompareBool(_) => NonNegative, + BinaryOpKind::Arithmetic(op) => arithmetic_sign(op, signs[value], rhs_value), + }; + } + signs.resize(output.fields.len(), Unknown); + signs + } + } +} + +fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { + use ASAPOp::*; + match op { + SummaryAgg { + child, + summary_type, + input, + reduction, + .. + } => state_signs(child, summary_type, input, reduction) + .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs), + // The build's layout without its own time column, after the + // computed `window_end` (column 0). + SummaryTumblingWindowChain { + child, + summary_type, + input, + reduction, + .. + } => match state_signs(child, summary_type, input, reduction) { + Some((build, mut signs)) => { + if let Some(time) = build.time_index { + signs.remove(time); + } + signs.insert(0, Unknown); + signs + } + None => vec![Unknown; output.fields.len()], + }, + SummaryMerge { children, group_by } => { + // The windows a `TimeRange` over a chain selects merge into the + // chain's build over that range. + if let (None, [child]) = (group_by, children.as_slice()) { + if let Some(SummaryTumblingWindowChain { + child, + summary_type, + input, + reduction, + .. + }) = summary_build(child) + { + return state_signs(&child, &summary_type, &input, &reduction) + .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs); + } + } + let merged = agreed(&children.iter().map(signs_of).collect::>()); + match group_by { + None => merged, + Some(keys) => { + let state = children[0] + .schema() + .fields + .iter() + .position(|field| !field.is_plain()); + keys.iter() + .chain(state.as_ref()) + .map(|&column| merged.get(column).copied().unwrap_or(Unknown)) + .collect() + } + } + } + SummaryEstimate { + summary_input, + query, + } => readout_signs(summary_input, output, |state, summary_type| match query { + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy => NonNegative, + SketchStatistic::Quantile { q } => quantile_sign(*q, state), + // A count read from a state whose counts may flip the weights' + // sign has none. + SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } + if SummaryTypeCapability::point_counts_may_flip_sign(summary_type) => + { + Unknown + } + SketchStatistic::PointCount { .. } | SketchStatistic::TopK { .. } => state, + }), + FinalizeExactAccumulator { child } => { + let counts = child.schema().fields.iter().any(|field| { + matches!( + field.dtype, + FieldDataType::ExactAggregate(ExactKind::Count, _) + ) + }); + readout_signs( + child, + output, + |state, _| if counts { NonNegative } else { state }, + ) + } + MaintainPopulation { child, .. } => signs_of(child), + EvaluatePopulation { child, .. } if output == child.schema() => signs_of(child), + EvaluatePopulation { .. } + | SummarySubtract { .. } + | SummaryDelete { .. } + | SummaryJoin { .. } + | Extension { .. } => vec![Unknown; output.fields.len()], + } +} + +/// A column item keeps its input column's sign; an expression item's sign +/// is [`expression_sign`]'s. +fn project_signs(cols: &[ProjectItem], input: &[ValueSign]) -> Vec { + cols.iter() + .map(|item| expression_sign(&item.expr, input)) + .collect() +} + +/// `abs` is non-negative, a negation flips its operand's sign, a numeric +/// cast keeps it, a numeric literal has its own, and arithmetic follows +/// [`arithmetic_sign`]; any other expression is `Unknown`. +fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { + match expr { + ScalarExpr::Column(column) => input.get(*column).copied().unwrap_or(Unknown), + ScalarExpr::Literal(ScalarValue::Float64(value)) => sign_of_number(*value), + ScalarExpr::Literal(ScalarValue::Int64(value)) => sign_of_number(*value as f64), + ScalarExpr::Negative { expr, .. } => flipped(expression_sign(expr, input)), + ScalarExpr::Cast { expr, .. } => expression_sign(expr, input), + ScalarExpr::Arithmetic { + op, left, right, .. + } => arithmetic_sign( + op, + expression_sign(left, input), + expression_sign(right, input), + ), + // PromQL's `abs(v)` lowers to `promql_abs`, SQL's to `abs`. + ScalarExpr::FunctionCall { name, .. } + if name.eq_ignore_ascii_case("abs") || name == "promql_abs" => + { + NonNegative + } + _ => Unknown, + } +} + +/// An aggregate's, by where each output column comes from +/// ([`aggregate_output`]): a kept input column keeps its sign, a +/// measure's value has [`measure_sign`] of its input column's, and a label +/// a measure adds is `Unknown`. +fn aggregate_signs( + child: &Input<'_>, + reduction: &Reduction, + measures: &[AggIntent], + output: &Schema, +) -> Vec { + let Ok((_, sources)) = aggregate_output(child.schema(), reduction, measures, &[]) else { + return vec![Unknown; output.fields.len()]; + }; + signs_from_sources(child, &sources, |index, input| { + measure_sign(&measures[index], input) + }) +} + +/// The sign of each output column from `sources`: an input column keeps +/// its sign, measure `index` has `measure(index, the sign of its input +/// column)`, and an added label is `Unknown`. +fn signs_from_sources( + child: &Input<'_>, + sources: &[OutputColumnSource], + measure: impl Fn(usize, ValueSign) -> ValueSign, +) -> Vec { + let input = &child.group_properties().column_signs; + let sign_of = |column: Option| { + column + .and_then(|column| input.get(column).copied()) + .unwrap_or(Unknown) + }; + sources + .iter() + .map(|source| match source { + OutputColumnSource::InputColumn(column) => sign_of(Some(*column)), + OutputColumnSource::Measure { + index, + input_column, + } => measure(*index, sign_of(*input_column)), + OutputColumnSource::AddedLabel => Unknown, + }) + .collect() +} + +/// The sign of `measure`'s value over an input value of sign `input`. +fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { + use AggIntent::*; + match measure { + Count + | Cardinality { .. } + | CountValues { .. } + | Changes + | Resets + | Group + | Absent { .. } + | AbsentOverTime + | PresentOverTime + | StdDev { .. } + | Variance { .. } + | FrequencyL2 { .. } + | FrequencyEntropy { .. } + | Math(crate::ir::operator::MathFunc::Abs) => NonNegative, + Sum { .. } | Avg { .. } | Min { .. } | Max { .. } | LastOverTime | FirstOverTime => input, + Quantile { q, .. } => quantile_sign(*q, input), + // A counter's rate and increase are non-negative; over a value of + // another sign a decrease reads as a reset, so no sign is known. + Rate | IRate | Increase if input == NonNegative => NonNegative, + _ => Unknown, + } +} + +/// A summary state's schema and signs, in its layout +/// ([`summary_agg_layout`]): its key columns keep their input's sign and its +/// state column records the sign of the values it is updated with; `None` +/// when the layout does not derive. +fn state_signs( + child: &Input<'_>, + summary_type: &FieldDataType, + update: &SummaryUpdate, + reduction: &Reduction, +) -> Option<(Schema, Vec)> { + let (schema, sources) = summary_agg_layout(child.schema(), summary_type, reduction).ok()?; + let updated_with = match &update.weight { + SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) + .ok() + .and_then(|column| signs_of(child).get(column).copied()) + .unwrap_or(Unknown), + SummaryInputExpr::Constant(value) => sign_of_number(*value), + SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, + }; + let signs = signs_from_sources(child, &sources, |_, _| updated_with); + Some((schema, signs)) +} + +/// A readout of summary state: the state's plain columns keep their sign +/// in place, and the value read out has `value(the state's recorded sign, +/// the state column's type)`. +/// `Unknown` everywhere when the output does not keep the state's layout (a +/// top-k's rows). +fn readout_signs( + state: &Input<'_>, + output: &Schema, + value: impl Fn(ValueSign, &FieldDataType) -> ValueSign, +) -> Vec { + let input = state.schema(); + if input.fields.len() != output.fields.len() { + return vec![Unknown; output.fields.len()]; + } + signs_of(state) + .into_iter() + .zip(&input.fields) + .map(|(sign, field)| { + if field.is_plain() { + sign + } else { + value(sign, &field.dtype) + } + }) + .collect() +} + +/// The sign of `left op right`: a sum of two values of one sign has that +/// sign; a product or quotient of two values of one sign is non-negative, +/// of opposite signs non-positive; any other operator, or an operand of +/// unknown sign, gives `Unknown`. PromQL's vector arithmetic and scalar +/// expressions both read it. +fn arithmetic_sign(op: &ArithmeticOpKind, left: ValueSign, right: ValueSign) -> ValueSign { + match (op, left, right) { + (ArithmeticOpKind::Add, NonNegative, NonNegative) => NonNegative, + (ArithmeticOpKind::Add, NonPositive, NonPositive) => NonPositive, + (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonNegative, NonNegative) + | (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonPositive, NonPositive) => NonNegative, + (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonNegative, NonPositive) + | (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, NonPositive, NonNegative) => NonPositive, + _ => Unknown, + } +} + +/// A quantile's sign: the input's for 0 ≤ q ≤ 1; PromQL gives ±Inf +/// outside it, so `Unknown`. +fn quantile_sign(q: f64, input: ValueSign) -> ValueSign { + if (0.0..=1.0).contains(&q) { + input + } else { + Unknown + } +} + +/// Per column, the inputs' sign if they all agree, else `Unknown`. +fn agreed(inputs: &[Vec]) -> Vec { + let Some((first, rest)) = inputs.split_first() else { + return Vec::new(); + }; + first + .iter() + .enumerate() + .map(|(column, &sign)| { + rest.iter().fold(sign, |agreed, other| { + agree(agreed, other.get(column).copied().unwrap_or(Unknown)) + }) + }) + .collect() +} + +fn agree(left: ValueSign, right: ValueSign) -> ValueSign { + if left == right { + left + } else { + Unknown + } +} + +fn flipped(sign: ValueSign) -> ValueSign { + match sign { + NonNegative => NonPositive, + NonPositive => NonNegative, + Unknown => Unknown, + } +} + +/// The sign of a number: its sign bit; `Unknown` for NaN. +fn sign_of_number(value: f64) -> ValueSign { + if value.is_nan() { + Unknown + } else if value.is_sign_negative() { + NonPositive + } else { + NonNegative + } +} diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index d00f3f0a6..86039fc38 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -1,11 +1,16 @@ //! Group scope: memo groups, the distinct logical sub-expressions of the //! workload, and the properties every candidate of a group shares. +use std::collections::BTreeSet; + +use super::column_properties::{column_signs, ValueSign}; +use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; use crate::ir::operator::input::OperatorInput; -use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::properties::AggregateCoverage; use crate::ir::schema::Schema; use crate::ir::ASAPOp; use crate::ir::SchemaDerivationError; +use crate::ir::{NonASAPOp, ScalarExpr}; use crate::ir::{Operator, OperatorResultKind}; use crate::stages::{CandidateForMemoGroup, Memo}; @@ -25,11 +30,15 @@ pub struct MemoGroup { /// changes. pub group_properties: GroupProperties, /// The groups that read it ([`Self::parent_groups`]) through their - /// expression or a candidate's. - pub parents: Vec, + /// expression or a candidate's: kept up to date by + /// [`Memo::find_or_add_group`](super::Memo::find_or_add_group) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). + pub(super) parents: Vec, /// The queries whose roots reach it ([`Self::reached_by_queries`]), - /// ascending. - pub queries: Vec, + /// ascending: kept up to date by + /// [`Memo::add_query`](super::Memo::add_query) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). + pub(super) queries: Vec, /// Its candidates: empty after Stage 0, filled by Stage 1. pub candidates: Vec, } @@ -53,81 +62,118 @@ impl MemoGroup { pub struct GroupProperties { pub schema: Schema, pub result_kind: OperatorResultKind, - /// Per output column, in schema order: the sign of its values. - pub column_sign: Vec, + /// Per output column, in schema order: the sign of its values, from the + /// input groups' and, for a `Scan`, the declared ones. + pub column_signs: Vec, /// Whether the result changes with the evaluation time: true for a /// group under a `TimeRange` relative to the evaluation time - /// (`sum_over_time(m[5m])`), false for one under a fixed `@ T` or - /// reading no time. A result that does not depend on it is the same at - /// every evaluation, so a stored copy delivers on every grid. + /// (`sum_over_time(m[5m])`) or reading it (`time()`, `now()`), false + /// for one under a fixed `@ T` or reading no time. A result that does + /// not depend on it is the same at every evaluation, so a stored copy + /// delivers on every grid. pub depends_on_evaluation_time: bool, - /// What an aggregate group covers: a summary state, a chain, a merge - /// over a chain's windows, or an exact `Aggregate` result; `None` for - /// any other group. Derived today for summary states only; exact - /// `Aggregate` groups follow in #1207 PR 2. - pub aggregate_coverage: Option, + /// What an aggregate group covers: a summary state, a chain, a + /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact + /// `Aggregate` result; `None` for any other group. + pub aggregate_coverage: Option>, /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). pub shareable_when_equal: bool, + /// The sources the `Scan`s below this group read: a `Scan`'s own, else + /// the union over its input groups. + pub scanned_sources: BTreeSet, } -/// The sign of one column's values, a group property derived from the -/// input groups like lineage (concepts doc, section 3, "Value sign"); a -/// `Scan`'s are declared per column in the data workload's source facts. -/// Error contracts that hold only for values of one sign read it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ValueSign { - NonNegative, - NonPositive, - Unknown, +/// Whether `operator` is a valid expression over its input groups +/// ([`MemoGroupAsInput`]), never a node tree: its output schema derives, its +/// inputs pass validation, and a merge reads only states it may merge. +/// Returns the output schema and, for a merge, the coverage its check +/// derived. +pub(super) fn validate_expression_over_input_groups( + operator: &Operator>, +) -> Result<(Schema, Option>), SchemaDerivationError> { + let schema = operator.output_schema()?; + operator.validate_inputs()?; + // A merge is valid only over states of one definition it may merge. + let merge_coverage = match operator { + Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(operator.aggregate_coverage()?), + _ => None, + }; + Ok((schema, merge_coverage)) } -/// The properties of a group, or a candidate, computing `expression` over -/// its input groups: the rules that check operator nodes, reading each -/// input group's `group_properties` and `expression` -/// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression -/// breaks them (schema derivation, input validation, merge check). -// `column_sign` and `depends_on_evaluation_time` are derived in #1207 PR 2; -// until then the result is unreachable. -#[allow(unreachable_code)] +/// The properties of a group computing `expression` over its input +/// groups, once it is valid over them +/// ([`validate_expression_over_input_groups`]), reading each input group's +/// `group_properties` and `expression`. pub(super) fn derive_group_properties( memo: &Memo, expression: &Operator, ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); - let schema = operator.output_schema()?; - operator.validate_inputs()?; - let aggregate_coverage = match SummaryCoverage::derive_from_inputs(&operator) { - Ok(coverage) => Some(coverage), - // A merge is valid only over states of one definition it may merge. - Err(error) if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) => { - return Err(error.into()) - } - Err(_) => None, - }; + let (schema, merge_coverage) = validate_expression_over_input_groups(&operator)?; + let aggregate_coverage = merge_coverage.or_else(|| operator.aggregate_coverage().ok()); Ok(GroupProperties { + column_signs: column_signs(&operator, &schema, &memo.source_facts), + depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), - column_sign: todo!("#1207 PR 2: each output column's sign, per operator, from the input groups' and the sources' declared signs"), - depends_on_evaluation_time: todo!("#1207 PR 2: whether the result changes with the evaluation time, from the operator and the input groups"), + scanned_sources: operator.scanned_sources(), }) } +/// Whether `operator`'s result changes with the evaluation time: a +/// `TimeRange` reads the window before it, and `time()` or `now()` read it +/// directly; `@ T` pins its subtree to T. Any other operator depends on it +/// when an input group does. +fn depends_on_evaluation_time(operator: &Operator>) -> bool { + fn reads_evaluation_time(expr: &ScalarExpr) -> bool { + matches!( + expr, + ScalarExpr::EvalTimestamp | ScalarExpr::CurrentTimestamp + ) || expr.children().into_iter().any(reads_evaluation_time) + } + let reads_it = match operator { + Operator::NonASAP(NonASAPOp::TimeRange { .. }) => return true, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: + TimeShift { + at: Some(AtModifier::Timestamp(_)), + .. + }, + .. + }) => return false, + Operator::NonASAP(op) => op.scalar_exprs().into_iter().any(reads_evaluation_time), + Operator::ASAP( + ASAPOp::SummaryAgg { filter, .. } | ASAPOp::SummaryTumblingWindowChain { filter, .. }, + ) => filter.iter().any(|pred| reads_evaluation_time(&pred.0)), + Operator::ASAP(_) => false, + }; + reads_it + || operator + .children() + .into_iter() + .any(|input| input.group_properties().depends_on_evaluation_time) +} + /// A memo group read as an operator's input: its `group_properties` and /// its `expression`, whose inputs are read the same way. #[derive(Clone, Copy)] -pub(crate) struct MemoGroupAsInput<'m> { - pub(crate) memo: &'m Memo, - pub(crate) group_id: MemoGroupId, +pub(super) struct MemoGroupAsInput<'m> { + pub(super) memo: &'m Memo, + pub(super) group_id: MemoGroupId, } impl<'m> MemoGroupAsInput<'m> { fn group(&self) -> &'m MemoGroup { &self.memo.groups()[self.group_id.0 as usize] } + + pub(super) fn group_properties(&self) -> &'m GroupProperties { + &self.group().group_properties + } } impl OperatorInput for MemoGroupAsInput<'_> { @@ -146,7 +192,17 @@ impl OperatorInput for MemoGroupAsInput<'_> { }) } - fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + type Reference = MemoGroupId; + + fn reference(&self) -> Option { + Some(self.group_id) + } + + fn aggregate_coverage(&self) -> Option<&AggregateCoverage> { self.group().group_properties.aggregate_coverage.as_ref() } + + fn scanned_sources(&self) -> BTreeSet { + self.group().group_properties.scanned_sources.clone() + } } diff --git a/crates/types/src/stages/logical/memo/hash_consing.rs b/crates/types/src/stages/logical/memo/hash_consing.rs new file mode 100644 index 000000000..61c489187 --- /dev/null +++ b/crates/types/src/stages/logical/memo/hash_consing.rs @@ -0,0 +1,74 @@ +//! The one equality that makes two expressions one memo group, and two +//! candidates of a group one candidate: typed equality of the operator, its +//! arguments and input groups, confirmed by equal serializations +//! ([`same_value`]; [`same_serialized_expression`] with each side's +//! serialization computed once). [`Memo::find_or_add_group`] looks a group up +//! by it, Stage 0 included, and [`Memo::add_candidate`] finds a duplicate by +//! it. Parameters θ are not an argument of any operator (they are a +//! candidate's `LogicalProperties`), so builds of one family over one input +//! that differ only in θ are one group. +//! +//! [`Memo::find_or_add_group`]: super::Memo::find_or_add_group +//! [`Memo::add_candidate`]: super::Memo::add_candidate + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use super::MemoGroupId; +use crate::ir::Operator; +use crate::stages::CandidateForMemoGroup; + +/// The canonical JSON of `expression` (operator, arguments and input +/// groups), serialized once per expression: what [`expression_hash`] hashes +/// and [`same_serialized_expression`] compares. +pub(super) fn serialized_expression(expression: &Operator) -> String { + serde_json::to_string(expression).expect("an expression over memo groups serializes") +} + +/// A hash consistent with [`same_serialized_expression`], used only to +/// narrow the groups compared: SipHash over the expression's +/// [`serialized_expression`]. A non-finite `f64` serializes as `null`, +/// which only widens one bucket; [`same_serialized_expression`] decides. +pub(super) fn expression_hash(serialized: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + serialized.hash(&mut hasher); + hasher.finish() +} + +/// Whether `left` and `right` are one expression: typed equality (operator, +/// arguments and input groups) confirmed by equal serializations (each +/// side's [`serialized_expression`], computed once by the caller), as +/// [`same_value`] decides it. +pub(super) fn same_serialized_expression( + (left, left_serialized): (&Operator, &str), + (right, right_serialized): (&Operator, &str), +) -> bool { + left == right && left_serialized == right_serialized +} + +/// Typed equality, confirmed by equal serializations: numeric `PartialEq` +/// alone conflates signed zeros, and `SketchParams`' equality omits the +/// recorded error and failure probability. The serialized check is +/// additional evidence, never a replacement for typed equality (JSON maps +/// non-finite floats to `null`). +pub(super) fn same_value(left: &T, right: &T) -> bool { + left == right + && match (serde_json::to_string(left), serde_json::to_string(right)) { + (Ok(left), Ok(right)) => left == right, + _ => false, + } +} + +/// Whether `left` and `right` are one candidate of a group: the same +/// expression and logical properties (parameters), each by [`same_value`]: +/// typed equality first, serialized only when typed-equal. +pub(super) fn same_expression_and_logical_properties( + left: &CandidateForMemoGroup, + right: &CandidateForMemoGroup, +) -> bool { + same_value(&left.expression.operator, &right.expression.operator) + && same_value( + &left.logical_properties.parameters, + &right.logical_properties.parameters, + ) +} diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index cdd7e212d..647cfa18a 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -2,9 +2,13 @@ //! over its groups; the operations over the whole memo //! (concepts doc `memo-groups-properties-requirements.md`, section 3; #1203). -use std::collections::{BTreeSet, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use super::group::derive_group_properties; +mod hash_consing; + +use super::group::{ + derive_group_properties, validate_expression_over_input_groups, MemoGroupAsInput, +}; use crate::accuracy::AccuracyTarget; use crate::ir::Operator; use crate::ir::QueryRoot; @@ -13,17 +17,30 @@ use crate::stages::{ CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalRequirements, MemoGroup, MemoGroupId, }; +use crate::workload::{DataWorkload, SourceFacts}; +use hash_consing::{ + expression_hash, same_expression_and_logical_properties, same_serialized_expression, + serialized_expression, +}; /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with /// its candidates ([`MemoGroup::candidates`]: empty after Stage 0), and each /// query's root over them. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct Memo { /// The groups, by id ([`Self::groups`]); added only through /// [`Self::find_or_add_group`]. groups: Vec, - /// The queries, by index in the workload. - pub queries: Vec, + /// The queries, by index in the workload ([`Self::queries`]); added only + /// through [`Self::add_query`]. + queries: Vec, + /// The groups [`Self::find_or_add_group`] may return, by + /// [`expression_hash`] of their expression: every group whose sharing + /// is legal ([`GroupProperties::shareable_when_equal`](crate::stages::GroupProperties::shareable_when_equal)). + groups_by_expression_hash: HashMap>, + /// The data workload's facts per source (`DataWorkload::sources`), fixed + /// at construction: a `Scan` group's column signs read them. + pub(super) source_facts: BTreeMap, } /// One query of the memo: its root over groups, its accuracy target and @@ -41,43 +58,189 @@ pub struct MemoQuery { } impl Memo { + /// A memo without groups over the sources `data` describes: a `Scan` + /// group's column signs are the ones `data` declares for its source + /// (`GroupProperties::column_signs`). + pub fn new(data: &DataWorkload) -> Self { + Self { + groups: Vec::new(), + queries: Vec::new(), + groups_by_expression_hash: HashMap::new(), + source_facts: data.sources.clone(), + } + } + /// The groups, by id. pub fn groups(&self) -> &[MemoGroup] { &self.groups } + /// The queries, by index in the workload. + pub fn queries(&self) -> &[MemoQuery] { + &self.queries + } + /// Adds `query` and returns its index: its root's groups and every /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 /// adds each query once. pub fn add_query(&mut self, query: MemoQuery) -> usize { - let _ = query; - todo!("#1207 PR 2") + let index = self.queries.len(); + let groups = query.root.operator_refs().into_iter().copied().collect(); + self.queries.push(query); + self.add_queries_below(groups, &[index]); + index } - /// Hash-consing: the group with this expression, added if new, with the - /// `GroupProperties` [`derive_group_properties`] derives. Fails when the - /// expression breaks the node rules (schema derivation, input - /// validation, merge check). + /// Hash-consing: the group whose expression is `expression` (operator, + /// arguments and input groups by typed equality, confirmed by equal + /// serializations: `same_serialized_expression`), else a + /// new group for it, with the `GroupProperties` `derive_group_properties` + /// derives from its input groups, listed as a parent of each input + /// group. A group whose copies may return different rows (an unordered + /// `LIMIT`, [`GroupProperties::shareable_when_equal`](crate::stages::GroupProperties::shareable_when_equal)) is never returned + /// for another expression, so such an expression always gets a new + /// group. Fails, leaving the memo unchanged, when the expression is not + /// valid over its input groups (its output schema does not derive, an + /// input fails validation, or a merge reads states it may not merge). pub fn find_or_add_group( &mut self, expression: Operator, ) -> Result { - let _group_properties = derive_group_properties(self, &expression)?; - todo!("#1207 PR 2: hash-consing over groups") + let serialized = serialized_expression(&expression); + let hash = expression_hash(&serialized); + // Only shareable groups are listed by hash, and an equal expression + // is as shareable as the group it equals, which was validated. + if let Some((group, _)) = self + .groups_by_expression_hash + .get(&hash) + .and_then(|groups| { + groups.iter().find(|(group, known)| { + same_serialized_expression( + (&self.groups[group.0 as usize].expression, known), + (&expression, &serialized), + ) + }) + }) + { + return Ok(*group); + } + let group_properties = derive_group_properties(self, &expression)?; + let shareable = group_properties.shareable_when_equal; + let id = MemoGroupId( + u32::try_from(self.groups.len()).expect("a memo has at most u32::MAX groups"), + ); + for &input in expression.children() { + add_parent(&mut self.groups[input.0 as usize], id); + } + self.groups.push(MemoGroup { + expression, + group_properties, + parents: Vec::new(), + queries: Vec::new(), + candidates: Vec::new(), + }); + if shareable { + self.groups_by_expression_hash + .entry(hash) + .or_default() + .push((id, serialized)); + } + Ok(id) } - /// Adds `candidate` to `group`'s candidates. Fails when its expression - /// breaks the node rules ([`derive_group_properties`]) or its output - /// schema or kind differs from the group's. + /// Adds `candidate` to `group`'s candidates and returns its id, unless + /// the group already has a candidate with the same expression + /// (`same_expression`) and logical properties: then that candidate's + /// id, and `candidate` is dropped. A new candidate makes `group` a parent + /// of each of its input groups, and `group`'s queries reach each input + /// group and every group below it. + /// + /// Fails, leaving the memo unchanged, when the candidate's expression + /// is not valid over its input groups + /// (`validate_expression_over_input_groups`), + /// or when its output schema or kind differs from the group's: then it + /// does not compute the group's result. pub fn add_candidate( &mut self, group: MemoGroupId, candidate: CandidateForMemoGroup, ) -> Result { - let _output = derive_group_properties(self, &candidate.expression.operator) - .map_err(CandidateError::Schema)?; - let _ = group; - todo!("#1207 PR 2") + if let Some(index_in_group) = self.groups[group.0 as usize] + .candidates + .iter() + .position(|known| same_expression_and_logical_properties(known, &candidate)) + { + return Ok(CandidateForMemoGroupId { + group, + index_in_group, + }); + } + let operator = candidate + .expression + .operator + .map_children(|&group_id| MemoGroupAsInput { + memo: self, + group_id, + }); + let (schema, _) = validate_expression_over_input_groups(&operator) + .map_err(CandidateError::NotValidOverInputGroups)?; + let result_kind = operator.output_kind(); + let group_properties = &self.groups[group.0 as usize].group_properties; + if schema != group_properties.schema || result_kind != group_properties.result_kind { + return Err(CandidateError::OutputDiffersFromGroup { group }); + } + let candidates = &mut self.groups[group.0 as usize].candidates; + let input_groups: Vec = candidate + .expression + .operator + .children() + .into_iter() + .copied() + .collect(); + candidates.push(candidate); + let index_in_group = candidates.len() - 1; + for &input in &input_groups { + add_parent(&mut self.groups[input.0 as usize], group); + } + let queries = self.groups[group.0 as usize].queries.clone(); + self.add_queries_below(input_groups, &queries); + Ok(CandidateForMemoGroupId { + group, + index_in_group, + }) + } + + /// Adds `queries` to each of `groups` and every group below them, through + /// the groups' expressions and their candidates' input groups. A group + /// that has every query already is not walked: every group below it has + /// them too. + fn add_queries_below(&mut self, groups: Vec, queries: &[usize]) { + let mut to_visit = groups; + while let Some(group) = to_visit.pop() { + let group = &mut self.groups[group.0 as usize]; + let mut added = false; + for &query in queries { + if let Err(position) = group.queries.binary_search(&query) { + group.queries.insert(position, query); + added = true; + } + } + if added { + to_visit.extend( + group + .expression + .children() + .into_iter() + .chain( + group + .candidates + .iter() + .flat_map(|candidate| candidate.expression.operator.children()), + ) + .copied(), + ); + } + } } /// The candidate `id`. @@ -146,19 +309,31 @@ impl Memo { } } -/// The groups some query's root reads, ascending: the groups of the -/// `queries`' roots that are a query's result or a scalar query's operand. -pub fn root_groups(queries: &[MemoQuery]) -> BTreeSet { - queries - .iter() - .flat_map(|query| query.root.operator_refs()) - .copied() - .collect() +/// Lists `parent` among `group`'s parents, once. +fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { + if !group.parents.contains(&parent) { + group.parents.push(parent); + } } -/// The queries whose root reads `group`, ascending. -pub fn queries_rooted_at(queries: &[MemoQuery], group: MemoGroupId) -> Vec { - (0..queries.len()) - .filter(|&query| queries[query].root.operator_refs().contains(&&group)) - .collect() +impl Memo { + /// The groups some query's root reads, ascending: the groups of the + /// queries' roots that are a query's result or a scalar query's operand. + pub fn root_groups(&self) -> BTreeSet { + self.queries + .iter() + .flat_map(|query| query.root.operator_refs()) + .copied() + .collect() + } + + /// The queries whose root reads `group`, ascending. + pub fn queries_rooted_at(&self, group: MemoGroupId) -> Vec { + (0..self.queries.len()) + .filter(|&query| self.queries[query].root.operator_refs().contains(&&group)) + .collect() + } } + +#[cfg(test)] +mod tests; diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs new file mode 100644 index 000000000..5e436c7a9 --- /dev/null +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -0,0 +1,584 @@ +//! The `Memo` methods: hash-consing groups ([`Memo::find_or_add_group`]) and +//! storing candidates ([`Memo::add_candidate`]), on the concepts doc's KLL +//! example (section 1: G1 the latency scan, G2 its KLL state). + +use super::*; +use crate::ir::operator::arguments::{GroupKeys, Reduction, Source}; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use crate::ir::scalar::{ProjectItem, SortKey}; +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; +use crate::stages::logical::MemoGroupAsInput; +use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; +use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; +use crate::test_support::memo_fixtures::{aggregate_where, readout, scan, state}; + +/// An unordered `LIMIT 10` over `input`, whose copies may return +/// different rows. +fn unordered_limit(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: input, + }) +} + +/// `Project(input)` keeping its first column. +fn project_first_column(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![crate::ir::ProjectItem { + alias: None, + expr: crate::ir::ScalarExpr::Column(0), + }], + qualifier: None, + child: input, + }) +} + +/// A candidate computing `operator` with `parameters`. +fn candidate( + operator: Operator, + parameters: Option, +) -> CandidateForMemoGroup { + let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; + CandidateForMemoGroup { + // Built directly: the doc example's k = 200 and k = 345 are not + // in today's KLL list, which `LogicalProperties::new` checks. + logical_properties: LogicalProperties { parameters }, + expression: CandidateExpression { operator }, + input_requirements, + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + } +} + +/// A query rooted at `group`, without an accuracy target or $n_q$. +fn query(group: MemoGroupId) -> MemoQuery { + MemoQuery { + root: QueryRoot::Operator(group), + accuracy_target: None, + number_of_values_per_query_evaluation: None, + } +} + +/// KLL `k`. +fn kll_k(k: u32) -> Option { + Some(SketchParams::Kll { k }) +} + +/// An equal expression over the same input groups is found, not added. +#[test] +fn find_or_add_group_returns_the_group_of_an_equal_expression() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + assert_eq!( + memo.find_or_add_group(scan("latency", &[], vec![])) + .unwrap(), + g1 + ); + assert_eq!( + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(), + g2 + ); + assert_ne!( + memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(), + g1 + ); + assert_eq!(memo.groups().len(), 3); + assert_eq!( + memo.groups()[g2.0 as usize].expression.children(), + vec![&g1] + ); +} + +/// The same operator over different input groups is a different group. +#[test] +fn find_or_add_group_separates_equal_operators_over_different_inputs() { + let mut memo = Memo::new(&DataWorkload::default()); + let latency = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let errors = memo.find_or_add_group(scan("errors", &[], vec![])).unwrap(); + assert_ne!( + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), latency)) + .unwrap(), + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), errors)) + .unwrap() + ); +} + +/// An expression whose copies may return different rows (an unordered +/// `LIMIT`) gets a new group every time. +#[test] +fn find_or_add_group_never_returns_an_unshareable_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + assert_ne!( + memo.find_or_add_group(unordered_limit(g1)).unwrap(), + memo.find_or_add_group(unordered_limit(g1)).unwrap() + ); +} + +/// A new group is a parent of each of its input groups, listed once. +#[test] +fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + memo.find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + assert_eq!(memo.groups()[g1.0 as usize].parent_groups(), &[g2]); + assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); +} + +/// KLL builds with k = 200 and k = 345 are two candidates of one group +/// G2: θ is a logical property, not part of the expression. +#[test] +fn kll_builds_with_k_200_and_k_345_share_one_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let k200_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let k200 = memo + .add_candidate( + k200_group, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(200), + ), + ) + .unwrap(); + let k345_group = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let k345 = memo + .add_candidate( + k345_group, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(345), + ), + ) + .unwrap(); + assert_eq!(k200_group, k345_group); + assert_ne!(k200, k345); + assert_eq!(memo.groups()[k200_group.0 as usize].candidates.len(), 2); + assert_eq!( + memo.candidate(k345).logical_properties.parameters(), + kll_k(345).as_ref() + ); +} + +/// A candidate with the same expression and logical properties as one +/// in the group is not added again; its id is returned. +#[test] +fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let first = memo + .add_candidate( + g2, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(200), + ), + ) + .unwrap(); + let again = memo + .add_candidate( + g2, + candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(200), + ), + ) + .unwrap(); + assert_eq!(first, again); + assert_eq!( + first, + CandidateForMemoGroupId { + group: g2, + index_in_group: 0 + } + ); + let other_size = candidate( + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1), + kll_k(345), + ); + assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); + assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); +} + +/// A new candidate makes its group a parent of each group it reads, and +/// the group's queries reach those groups and every group below them. +#[test] +fn add_candidate_maintains_parents_and_queries() { + // Query 0 is G3 = SummaryEstimate[q=0.95](G2) over G2 = KLL(G1). + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let g3 = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.95 }, g2)) + .unwrap(); + assert_eq!(memo.add_query(query(g3)), 0); + // A rule adds a state group G5 over another scan G4: no query + // reaches them yet. + let g4 = memo + .find_or_add_group(scan("latency_copy", &[], vec![])) + .unwrap(); + let g5 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g4)) + .unwrap(); + assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); + // G3 gains a candidate reading G5: G5 and G4 now serve query 0. + memo.add_candidate( + g3, + candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g5), None), + ) + .unwrap(); + assert_eq!(memo.groups()[g5.0 as usize].parent_groups(), &[g3]); + assert_eq!(memo.groups()[g4.0 as usize].parent_groups(), &[g5]); + assert_eq!(memo.groups()[g5.0 as usize].reached_by_queries(), &[0]); + assert_eq!(memo.groups()[g4.0 as usize].reached_by_queries(), &[0]); + // Query 1 is G6 = Project(G3): it reaches G3 and everything below + // it, through expressions and candidates, in ascending order. + let g6 = memo.find_or_add_group(project_first_column(g3)).unwrap(); + assert_eq!(memo.add_query(query(g6)), 1); + for group in [g1, g2, g3, g4, g5] { + assert_eq!( + memo.groups()[group.0 as usize].reached_by_queries(), + &[0, 1], + "group {group:?}" + ); + } + assert_eq!(memo.groups()[g3.0 as usize].parent_groups(), &[g6]); +} + +/// A candidate that computes another result than its group's (a +/// readout offered for a state group) is rejected and changes nothing. +#[test] +fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate(g2, candidate(readout(SketchStatistic::Quantile { q: 0.5 }, g2), None)), + Err(CandidateError::OutputDiffersFromGroup { group }) if group == g2 + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); +} + +/// A candidate whose expression fails input validation (a readout of +/// raw rows instead of summary state) is rejected and changes nothing. +#[test] +fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let g2 = memo + .find_or_add_group(state(FieldDataType::Sketch(SketchAlgorithm::Kll), g1)) + .unwrap(); + let g3 = memo + .find_or_add_group(readout(SketchStatistic::Quantile { q: 0.95 }, g2)) + .unwrap(); + let over_raw_rows = + readout(SketchStatistic::Quantile { q: 0.95 }, g1).map_children(|&group_id| { + MemoGroupAsInput { + memo: &memo, + group_id, + } + }); + assert!( + over_raw_rows.output_schema().is_ok(), + "fixture: the schema derives; only input validation rejects it" + ); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate( + g3, + candidate(readout(SketchStatistic::Quantile { q: 0.95 }, g1), None) + ), + Err(CandidateError::NotValidOverInputGroups(_)) + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); +} + +/// An expression with no output schema (a group-by column out of range) +/// is an error, and no group is added. +#[test] +fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo + .find_or_add_group(scan("latency", &[], vec![])) + .unwrap(); + let out_of_range = Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![9]), + measures: vec![crate::ir::operator::agg_intent::AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child: g1, + }); + assert!(memo.find_or_add_group(out_of_range).is_err()); + assert_eq!(memo.groups().len(), 1); + assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); +} + +/// `Scan m`: `[ts, service, value, latency]`, no unique key. +fn service_scan() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +/// Whether `a` and `b`, each built into one memo through +/// `find_or_add_group`, are one group. +fn one_group( + a: impl FnOnce(&mut Memo) -> MemoGroupId, + b: impl FnOnce(&mut Memo) -> MemoGroupId, +) -> bool { + let mut memo = Memo::new(&DataWorkload::default()); + let a = a(&mut memo); + let b = b(&mut memo); + a == b +} + +/// `operator` over the group of [`service_scan`]. +fn over_scan( + operator: impl FnOnce(MemoGroupId) -> Operator, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + |memo| { + let scan = memo.find_or_add_group(service_scan()).unwrap(); + memo.find_or_add_group(operator(scan)).unwrap() + } +} + +/// `quantile(q, col) by (by)` over [`service_scan`], with `filter`. +fn quantile_agg( + by: Vec, + col: Option, + q: f64, + filter: Option>, +) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + aggregate_where( + Reduction::by(by), + AggIntent::Quantile { col, q }, + filter, + child, + ) + }) +} + +/// Aggregates that differ in a column, a quantile or a filter are +/// different groups. +#[test] +fn aggregates_that_differ_in_an_argument_are_different_groups() { + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(3), 0.5, None) + )); + assert!(!one_group( + quantile_agg(vec![], Some(2), 0.5, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let latency_above_1 = Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(3)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Sql, + }); + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5, None), + quantile_agg(vec![1], Some(2), 0.5, Some(latency_above_1)) + )); +} + +/// Equal expressions without a unique key are one group when they return +/// the same rows each time: a keyless scan, a global aggregate, a +/// `without` aggregate. +#[test] +fn equal_keyless_expressions_that_return_the_same_rows_are_one_group() { + let table_scan = |memo: &mut Memo| { + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "metrics".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, true), + ], + 0, + vec![], + ), + })) + .unwrap() + }; + assert!(one_group(table_scan, table_scan)); + assert!(one_group( + quantile_agg(vec![], Some(2), 0.9, None), + quantile_agg(vec![], Some(2), 0.9, None) + )); + let without = || { + over_scan(|child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::without(vec![0])), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }) + }; + assert!(one_group(without(), without())); +} + +/// `DISTINCT` on every column returns the same rows each time and is one +/// group; a `Dedup` on some columns keeps an unspecified row per key. +#[test] +fn dedup_is_one_group_only_on_every_column() { + let dedup = |cols: Vec| { + over_scan(move |child| Operator::NonASAP(NonASAPOp::Dedup { cols, child })) + }; + assert!(one_group(dedup(vec![]), dedup(vec![]))); + assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); +} + +/// Two unordered `LIMIT 10`s may return different rows: two groups, and +/// so are equal parents over them. +#[test] +fn unordered_limits_and_their_parents_are_different_groups() { + assert!(!one_group( + over_scan(unordered_limit), + over_scan(unordered_limit) + )); + let distinct_over_limit = |memo: &mut Memo| { + let limit = over_scan(unordered_limit)(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Dedup { + cols: vec![], + child: limit, + })) + .unwrap() + }; + assert!(!one_group(distinct_over_limit, distinct_over_limit)); +} + +/// A `LIMIT` over a `Sort` with the same partitions keeps the first rows +/// of a fixed order (one group); over a `Sort` partitioned differently, +/// which rows come first depends on input order (two groups). +#[test] +fn limits_over_a_sort_with_their_partitions_are_one_group() { + let limit = |sort_partitions: GroupKeys| { + move |memo: &mut Memo| { + let sorted = over_scan(|child| { + Operator::NonASAP(NonASAPOp::Sort { + keys: vec![SortKey { + expr: ScalarExpr::Column(2), + ascending: false, + nulls_first: false, + }], + partition_by: sort_partitions, + child, + }) + })(memo); + memo.find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: sorted, + })) + .unwrap() + } + }; + assert!(one_group( + limit(GroupKeys::default()), + limit(GroupKeys::default()) + )); + assert!(!one_group( + limit(GroupKeys::by(vec![1])), + limit(GroupKeys::by(vec![1])) + )); +} + +/// A projection emitting the literal `value`. +fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { + over_scan(move |child| { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::Literal(ScalarValue::Float64(value)), + }], + qualifier: None, + child, + }) + }) +} + +/// Signed zeros and different non-finite literals are different groups; +/// JSON's `null` for non-finite floats never decides equality. +#[test] +fn signed_zero_and_non_finite_literals_are_different_groups() { + for (a, b) in [ + (0.0, -0.0), + (-0.0, 0.0), + (f64::INFINITY, f64::NEG_INFINITY), + (f64::NAN, f64::NAN), + ] { + assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); + } + assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); +} diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 249ca41cc..59ba58c11 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -1,20 +1,24 @@ //! The logical side: Stage 0's and Stage 1's memo, by scope (memo → group //! → candidate → candidate input → edge → Stage 1's output). +mod aggregate_coverage; mod candidate; +mod column_properties; mod edge; mod group; mod memo; mod memo_candidates; mod requirements; +pub use aggregate_coverage::view_coverage; pub use candidate::{ CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, }; +pub use column_properties::ValueSign; pub use edge::LogicalEdge; -pub(crate) use group::MemoGroupAsInput; -pub use group::{GroupProperties, MemoGroup, MemoGroupId, ValueSign}; -pub use memo::{queries_rooted_at, root_groups, Memo, MemoQuery}; +use group::MemoGroupAsInput; +pub use group::{GroupProperties, MemoGroup, MemoGroupId}; +pub use memo::{Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; pub use requirements::{LogicalRequirements, RequiredAccuracy}; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index fa095e9bc..a0385e635 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -12,9 +12,8 @@ mod per_computation_logical_memo_candidates; mod physical; mod pruning; -pub(crate) use logical::MemoGroupAsInput; pub use logical::{ - queries_rooted_at, root_groups, CandidateError, CandidateExpression, CandidateForMemoGroup, + view_coverage, CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 0aaf77cd4..502592071 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -110,6 +110,7 @@ mod tests { producer_satisfies_physical_requirements, CandidateForMemoGroupId, ExecutionContext, Lifetime, PhysicalProperties, }; + use crate::workload::DataWorkload; use crate::plan::materialized_view::RefreshWindow; @@ -161,9 +162,9 @@ mod tests { /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last /// 5 minutes, each with its own expression as its one candidate. Built - /// through `find_or_add_group` and `add_candidate` (#1207 PR 2). + /// through `find_or_add_group` and `add_candidate`. fn memo() -> Memo { - let mut memo = Memo::default(); + let mut memo = Memo::new(&DataWorkload::default()); let scan = memo .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { @@ -284,7 +285,7 @@ mod tests { /// the one evaluation its result belongs to. Neither reads the memo. #[test] fn delivers_on_by_lifetime_without_windows() { - let candidates = candidates(Memo::default(), vec![]); + let candidates = candidates(Memo::new(&DataWorkload::default()), vec![]); let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); @@ -310,7 +311,6 @@ mod tests { /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s /// serves both. #[test] - #[ignore = "filled in by #1207 PR 2 (#1230)"] fn delivers_on_a_chain_when_its_width_divides_the_context_times() { let candidates = candidates(memo(), vec![None; 5]); let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); @@ -349,7 +349,6 @@ mod tests { /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or /// another phase. Without a known grid it serves none. #[test] - #[ignore = "filled in by #1207 PR 2 (#1230)"] fn delivers_on_a_stored_result_on_its_group_grid() { let grid_60s = Some(EvaluationGrid { period_ms: 60_000, diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index 9d28aa324..c389921c2 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -86,7 +86,7 @@ pub enum PruneReason { /// candidates, once each candidate's allowed consumers are applied. ReachesNoQueryOutput, /// Stage 2: a Count-Min build (`Cms`, `CmsWithHeap`, `HydraCms`) whose - /// weight column's `column_sign` is not `NonNegative`: its estimates are + /// weight column's `column_signs` is not `NonNegative`: its estimates are /// bounds only for non-negative weights. CountMinWeightsNotProvenNonNegative, /// Stage 3: the candidate it names, in the same memo group with the diff --git a/crates/types/src/summary_type/capability.rs b/crates/types/src/summary_type/capability.rs index 612f854ae..b615192e9 100644 --- a/crates/types/src/summary_type/capability.rs +++ b/crates/types/src/summary_type/capability.rs @@ -110,6 +110,52 @@ impl SummaryTypeCapability { ) && !Self::shares_one_state_across_groups(summary_type) } + /// Whether a point count or top-k read from a state of `summary_type` + /// may have a sign the weights it was updated with do not: a median of + /// signed counters (Count Sketch, UnivMon and their variants) can be + /// negative over non-negative weights; an overestimating counter + /// (Count-Min) or an exact count cannot. Every algorithm is listed, so + /// one `asap_sketchlib` adds fails to compile here until it is placed. + /// `asap_sketchlib`'s `experimental` algorithms are not listed: the + /// planner does not build it with that feature, as `parameter_list` + /// does not. + pub(crate) fn point_counts_may_flip_sign(summary_type: &FieldDataType) -> bool { + use SketchAlgorithm as A; + let FieldDataType::Sketch(algorithm) = summary_type else { + return false; + }; + match algorithm { + A::CountSketch + | A::CountSketchFold + | A::CountSketchWithHeap + | A::CountSketchWithL2Norm + | A::CountSketchWithSpaceSaving + | A::HydraCountSketch + | A::UnivMon + | A::UnivMonQ + | A::UnivMonTwoTier + | A::HydraUnivMon => true, + A::Cms + | A::CmsWithHeap + | A::CmsWithSpaceSaving + | A::CmsFold + | A::HydraCms + | A::Hll + | A::HllHip + | A::Kmv + | A::Kll + | A::KllDynamic + | A::HydraKll + | A::HydraHll + | A::DDSketch + | A::SpaceSaving + | A::Elastic + | A::Coco + | A::Bloom + | A::SetAggregator => false, + } + } + /// Whether `summary_type` is a Hydra algorithm: one structure shared by /// every group, not one state per group. pub fn shares_one_state_across_groups(summary_type: &FieldDataType) -> bool { diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 3b3a314ae..60833bac8 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -4,9 +4,10 @@ //! to the crate-private function of the same name, except the fixture //! [`population_over_selector`], which the executor's tests build. +pub mod memo_fixtures; + use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; -use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::plan::materialized_view::RawDataWindow; use crate::ir::schema::Schema; @@ -14,11 +15,6 @@ use crate::ir::{ASAPOp, NonASAPOp}; use crate::ir::{OperatorNode, SchemaDerivationError}; use std::rc::Rc; -/// `SummaryCoverage::definition_id`. -pub fn definition_id(coverage: &SummaryCoverage) -> String { - coverage.definition_id() -} - /// `Operator::output_schema`. pub fn output_schema(operator: &Operator) -> Result { operator.output_schema() diff --git a/crates/types/src/test_support/memo_fixtures.rs b/crates/types/src/test_support/memo_fixtures.rs new file mode 100644 index 000000000..6fc9b4d40 --- /dev/null +++ b/crates/types/src/test_support/memo_fixtures.rs @@ -0,0 +1,108 @@ +//! Memo expressions over group ids that the memo's unit tests and this +//! crate's memo integration tests (`memo_group_properties`, +//! `memo_aggregate_coverage`) build their examples from. +use std::time::Duration; + +use crate::ir::operator::arguments::{AtModifier, Reduction, Source, TimeShift}; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::ColumnRef; +use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; +use crate::ir::{ASAPOp, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind}; +use crate::stages::MemoGroupId; +use crate::summary_type::{SketchStatistic, SummaryUpdate}; + +/// `Scan metric{predicates}`: `[ts, value, columns…]`, `ts` its time index. +pub fn scan( + metric: &str, + columns: &[(&str, DataType)], + predicates: Vec>, +) -> Operator { + let fields = [ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ] + .into_iter() + .chain( + columns + .iter() + .map(|(name, dtype)| Field::plain(*name, dtype.clone(), false)), + ) + .collect(); + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: predicates.into_iter().map(Predicate).collect(), + schema: Schema::with_time_index(fields, 0, vec![]), + }) +} + +pub fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + }) +} + +/// `TimeRange[width_ms]`: a PromQL range `[w]`. +pub fn range(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(width_ms as u64), + kind: TimeRangeKind::Range, + child, + }) +} + +/// `TimeShift(offset_ms, at)`: PromQL `offset` and `@`. +pub fn shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { offset_ms, at }, + child, + }) +} + +/// `SummaryAgg[summary_type]` of the sample value per series. +pub fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +/// `Aggregate(reduction; measure)` over `child`, over every row. +pub fn aggregate( + reduction: Reduction, + measure: AggIntent, + child: MemoGroupId, +) -> Operator { + aggregate_where(reduction, measure, None, child) +} + +/// `Aggregate(reduction; measure)` over `child`, the measure over the rows +/// `filter` keeps (`None`: every row). +pub fn aggregate_where( + reduction: Reduction, + measure: AggIntent, + filter: Option>, + child: MemoGroupId, +) -> Operator { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures: vec![measure], + output_names: vec![], + filters: filter.map_or_else(Vec::new, |filter| vec![Some(filter)]), + having: None, + child, + }) +} + +/// `SummaryEstimate[query](state)`: a readout of `state`. +pub fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query, + }) +} diff --git a/crates/types/src/workload/data.rs b/crates/types/src/workload/data.rs index 124580cd7..f7ea803bd 100644 --- a/crates/types/src/workload/data.rs +++ b/crates/types/src/workload/data.rs @@ -114,7 +114,7 @@ pub struct SourceFacts { pub metric_type: Option, /// The declared sign of each column's values, keyed by column name /// (default: none declared), e.g. `{"value": "non_negative"}` for a - /// latency gauge. A `Scan`'s `column_sign` reads it; an undeclared + /// latency gauge. A `Scan`'s `column_signs` reads it; an undeclared /// column is `Unknown`, except a PromQL counter's value, which is /// non-negative without a declaration. The deployment must ensure it. /// Omitted from JSON when empty. @@ -313,26 +313,7 @@ impl DataWorkload { &self, series: &str, ) -> Option<(&SourceFacts, Option)> { - use PrometheusMetricType::{Counter, Histogram, Summary}; - for (suffix, series_type) in [ - ("_bucket", None), - ("_sum", Some(Counter)), - ("_count", Some(Counter)), - ] { - let Some(family) = series.strip_suffix(suffix) else { - continue; - }; - let Some(facts) = self.sources.get(family) else { - continue; - }; - match (facts.metric_type, series_type) { - (Some(Histogram), None) => return Some((facts, Some(Histogram))), - (Some(Histogram | Summary), Some(counter)) => return Some((facts, Some(counter))), - _ => {} - } - } - let facts = self.sources.get(series)?; - Some((facts, facts.metric_type)) + series_facts(&self.sources, series) } /// Data of unknown arrival whose `metrics` are each sampled every @@ -427,6 +408,35 @@ impl DataWorkload { } } +/// [`DataWorkload::series_facts`] over `sources`, a data workload's +/// [`DataWorkload::sources`]: the facts of the source a PromQL selector over +/// `series` reads, and that series' own type. +pub(crate) fn series_facts<'a>( + sources: &'a BTreeMap, + series: &str, +) -> Option<(&'a SourceFacts, Option)> { + use PrometheusMetricType::{Counter, Histogram, Summary}; + for (suffix, series_type) in [ + ("_bucket", None), + ("_sum", Some(Counter)), + ("_count", Some(Counter)), + ] { + let Some(family) = series.strip_suffix(suffix) else { + continue; + }; + let Some(facts) = sources.get(family) else { + continue; + }; + match (facts.metric_type, series_type) { + (Some(Histogram), None) => return Some((facts, Some(Histogram))), + (Some(Histogram | Summary), Some(counter)) => return Some((facts, Some(counter))), + _ => {} + } + } + let facts = sources.get(series)?; + Some((facts, facts.metric_type)) +} + fn validate_optional_rate(rate: Option) -> Result<(), WorkloadError> { rate.map(validate_rate).transpose().map(|_| ()) } diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index 5b4f20270..2af8e5966 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -8,6 +8,7 @@ pub mod parsed_workload; mod planning; mod query; +pub(crate) use data::series_facts; pub use data::{ DataArrival, DataWorkload, LabelSetDistinctValues, PrometheusMetricType, Rate, SourceFacts, StreamNorms, diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index bf56559c0..ad9e48481 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,8 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod memo_aggregate_coverage; +mod memo_group_properties; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs new file mode 100644 index 000000000..7b57b5a6f --- /dev/null +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -0,0 +1,646 @@ +//! Aggregate coverage (`AggregateCoverage`), derived once per memo group +//! from its input groups (`docs/design_docs/proposals/asap-primitive-schema.md` +//! §4.2.2). The memo of the worked example: +//! +//! ```text +//! G1 Scan latency +//! G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) +//! G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) +//! G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) +//! ``` +use std::ops::Bound; + +use asap_types::ir::operator::arguments::{AtModifier, Reduction}; +use asap_types::ir::operator::AggIntent; + +use asap_types::ir::properties::selection::{ColumnIdentity, Constraint, IntervalConstraintLimit}; +use asap_types::ir::properties::{AggregateCoverage, CoverageError}; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, SchemaDerivationError, +}; +use asap_types::stages::{ + view_coverage, CandidateExpression, CandidateForMemoGroup, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; +use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; +use asap_types::workload::DataWorkload; + +const TS: usize = 0; +const JOB: usize = 2; +const A: usize = 3; +const B: usize = 4; +const MINUTE_MS: i64 = 60_000; +const HOUR_MS: i64 = 60 * MINUTE_MS; + +/// `Scan latency{predicates}`: `[ts, value, job, a, b]`. +fn latency(predicates: Vec>) -> Operator { + scan( + "latency", + &[ + ("job", DataType::Utf8), + ("a", DataType::Float64), + ("b", DataType::Float64), + ], + predicates, + ) +} + +fn job_is(job: &str) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(JOB)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(job.into()))), + semantics: ExprSemantics::Sql, + } +} + +/// `a + b > 10`: it spans two columns, so it is no selection. +fn a_plus_b_above_10() -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Add, + left: Box::new(ScalarExpr::Column(A)), + right: Box::new(ScalarExpr::Column(B)), + semantics: ExprSemantics::Sql, + }), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + } +} + +fn kll_summary_type() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) +} + +/// `SummaryAgg[KLL]` of the sample value per series. +fn kll(child: MemoGroupId) -> Operator { + state(kll_summary_type(), child) +} + +/// `SummaryTumblingWindowChain[KLL, width_ms]` of the sample value per series. +fn kll_chain(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + summary_type: kll_summary_type(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: width_ms as u64, + }) +} + +fn merge(children: Vec) -> Operator { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + }) +} + +/// KLL over the last minute of `G1` filtered by `pred`. +fn kll_of_last_minute_where( + memo: &mut Memo, + g1: MemoGroupId, + pred: ScalarExpr, +) -> MemoGroupId { + let filtered = memo.find_or_add_group(filter(pred, g1)).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, filtered)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() +} + +fn coverage(memo: &Memo, group: MemoGroupId) -> &AggregateCoverage { + memo.groups()[group.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state") +} + +/// `KLL` as an unlifted operator: its main input is `None`. +fn kll_unlifted() -> Operator> { + kll(MemoGroupId(0)).map_children(|_| None) +} + +fn jobs(values: &[&str]) -> Constraint { + Constraint::In( + values + .iter() + .map(|value| ScalarValue::Utf8((*value).into())) + .collect(), + ) +} + +/// `(t + lower_ms, t + upper_ms]` on the timestamp column, t the evaluation +/// time: left-open, right-closed like a PromQL range. +fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { + Constraint::Interval { + lower: Bound::Excluded(IntervalConstraintLimit::EvaluationTimePlusMs(lower_ms)), + upper: Bound::Included(IntervalConstraintLimit::EvaluationTimePlusMs(upper_ms)), + } +} + +fn last_minute() -> Constraint { + relative(-MINUTE_MS, 0) +} + +fn ts() -> ColumnIdentity { + ColumnIdentity::new(None, "ts") +} + +/// The constraint on the timestamp column of `group`'s one selection box. +fn time_of(memo: &Memo, group: MemoGroupId) -> Constraint { + let [selection] = coverage(memo, group).selection() else { + panic!("one box: {:?}", coverage(memo, group)) + }; + selection.columns()[&ts()].clone() +} + +/// G4: `job = "api"` and the minute are lifted; the definition is the KLL +/// over G1. +#[test] +fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input, Some(g1)); + assert_eq!( + coverage.definition().unlifted_operators, + vec![kll_unlifted()] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert_eq!(selection.columns()[&ts()], last_minute()); +} + +/// G7 has G4's definition (`==`) and a disjoint selection, so +/// `SummaryMerge(G4, G7)` is a valid group covering job ∈ {api, web}. +#[test] +fn states_with_equal_definitions_and_disjoint_selections_merge() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g7 = kll_of_last_minute_where(&mut memo, g1, job_is("web")); + assert_eq!( + coverage(&memo, g4).definition(), + coverage(&memo, g7).definition() + ); + let merged = memo.find_or_add_group(merge(vec![g4, g7])).unwrap(); + let coverage = coverage(&memo, merged); + assert_eq!(coverage.definition().input, Some(g1)); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// Two states that select the same rows do not merge: KLL counts a row +/// once per input. +#[test] +fn states_with_overlapping_selections_do_not_merge() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let groups = memo.groups().len(); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g4])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::Disjoint + } + )) + )); + assert_eq!(memo.groups().len(), groups); +} + +/// G10: `a + b > 10` spans two columns, so it stays as a `Filter` in the +/// definition; G10 does not merge with G4 although both select job = api. +#[test] +fn a_filter_that_is_no_selection_stays_unlifted() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g10 = kll_of_last_minute_where( + &mut memo, + g1, + ScalarExpr::BoolAnd(vec![job_is("api"), a_plus_b_above_10()]), + ); + let coverage_of_g10 = coverage(&memo, g10); + assert_eq!(coverage_of_g10.definition().input, Some(g1)); + assert_eq!( + coverage_of_g10.definition().unlifted_operators, + vec![ + kll_unlifted(), + filter(a_plus_b_above_10(), MemoGroupId(0)).map_children(|_| None), + ] + ); + assert_eq!( + coverage_of_g10.selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g10])), + Err(SchemaDerivationError::Coverage( + CoverageError::DefinitionMismatch + )) + )); +} + +/// `a > (SELECT threshold)`: a scalar subquery reading group `threshold`. +fn a_above(threshold: MemoGroupId) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(A)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::ScalarSubquery(threshold)), + semantics: ExprSemantics::Sql, + } +} + +/// A residual `Filter` that reads a scalar subquery names the subquery's +/// group (`Some`) next to its main input (`None`). Two states with that +/// equal definition and disjoint selections merge. +#[test] +fn states_whose_definitions_read_the_same_subquery_merge() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let threshold = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Values { + rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(10.0))]], + schema: Schema::new(vec![Field::plain("threshold", DataType::Float64, false)]), + })) + .unwrap(); + let state_of_job = |memo: &mut Memo, job: &str| { + let pred = ScalarExpr::BoolAnd(vec![job_is(job), a_above(threshold)]); + kll_of_last_minute_where(memo, g1, pred) + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input, Some(g1)); + assert_eq!( + definition.unlifted_operators, + vec![ + kll_unlifted(), + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(a_above(threshold).map_operator_refs(&mut |group| Some(*group))), + child: None, + }), + ] + ); + assert_eq!(definition, coverage(&memo, web).definition()); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// PromQL label matchers are the scan's predicates: they are lifted too, so +/// `latency{job="api"}` and `latency{job="web"}` states merge. The scan, +/// without them, is the last unlifted operator. +#[test] +fn scan_predicates_lift_and_their_states_merge() { + let mut memo = Memo::new(&DataWorkload::default()); + let state_of_job = |memo: &mut Memo, job: &str| { + let series = memo.find_or_add_group(latency(vec![job_is(job)])).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input, None); + assert_eq!( + definition.unlifted_operators, + vec![kll_unlifted(), latency(vec![]).map_children(|_| None)] + ); + assert_eq!( + coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// A chain has the definition of a build over its input and leaves the +/// timestamp column unconstrained. `TimeRange` and `TimeShift` over it +/// select windows as over raw rows, so the merge over that selection covers +/// what the direct build does, and `add_candidate` accepts the merge in the +/// direct build's group: +/// +/// ```text +/// G1 Scan latency +/// G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) (the query) +/// G5 Chain[KLL, 1m](G1); G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) +/// G4 candidate: SummaryMerge(G7) +/// ``` +#[test] +fn a_merge_over_a_chains_windows_covers_the_direct_build() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g2 = memo.find_or_add_group(shift(HOUR_MS, None, g1)).unwrap(); + let g3 = memo.find_or_add_group(range(5 * MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(kll(g3)).unwrap(); + let g5 = memo.find_or_add_group(kll_chain(MINUTE_MS, g1)).unwrap(); + let g6 = memo.find_or_add_group(shift(HOUR_MS, None, g5)).unwrap(); + let g7 = memo.find_or_add_group(range(5 * MINUTE_MS, g6)).unwrap(); + + let whole_chain = coverage(&memo, g5); + assert_eq!(whole_chain.definition(), coverage(&memo, g4).definition()); + assert!(whole_chain.selection()[0].columns().is_empty()); + let five_minutes_an_hour_ago = relative(-HOUR_MS - 5 * MINUTE_MS, -HOUR_MS); + assert_eq!(time_of(&memo, g7), five_minutes_an_hour_ago); + assert_eq!(coverage(&memo, g7), coverage(&memo, g4)); + + let expression = CandidateExpression { + operator: merge(vec![g7]), + }; + let merge_of_windows = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(g4, merge_of_windows).unwrap(); + let merged = memo.find_or_add_group(merge(vec![g7])).unwrap(); + assert_eq!(coverage(&memo, merged), coverage(&memo, g4)); +} + +/// The range and every offset lift into one interval relative to the +/// evaluation time t; `@ T` fixes the interval in absolute time: +/// +/// | Group | Expression | Timestamp column | +/// |---|---|---| +/// | G2 | `TimeRange[5m](G1)` (`m[5m]`) | `(t − 5m, t]` | +/// | G4 | `TimeRange[5m](TimeShift(1h)(G1))` (`m[5m] offset 1h`) | `(t − 65m, t − 60m]` | +/// | G6 | `TimeRange[5m](TimeShift(@ 1700000000)(G1))` | `(1699999700000, 1700000000000]` | +/// +/// each under a `SummaryAgg[KLL]`. +#[test] +fn time_range_and_shifts_lift_into_one_interval_on_the_timestamp_column() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let plain = kll_over_last_5m(&mut memo, g1); + let offset = kll_over_last_5m_of(&mut memo, shift(HOUR_MS, None, g1)); + let at_t = kll_over_last_5m_of( + &mut memo, + shift(0, Some(AtModifier::Timestamp(1_700_000_000_000)), g1), + ); + assert_eq!(time_of(&memo, plain), relative(-5 * MINUTE_MS, 0)); + assert_eq!(time_of(&memo, offset), relative(-3_900_000, -3_600_000)); + let timestamp = |ms| IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(ms)); + assert_eq!( + time_of(&memo, at_t), + Constraint::Interval { + lower: Bound::Excluded(timestamp(1_699_999_700_000)), + upper: Bound::Included(timestamp(1_700_000_000_000)), + } + ); + for state in [offset, at_t] { + assert_eq!( + coverage(&memo, state).definition(), + coverage(&memo, plain).definition() + ); + } +} + +/// `SummaryAgg[KLL](TimeRange[5m](G1))`: `m[5m]`. +fn kll_over_last_5m(memo: &mut Memo, input: MemoGroupId) -> MemoGroupId { + let ranged = memo.find_or_add_group(range(5 * MINUTE_MS, input)).unwrap(); + memo.find_or_add_group(kll(ranged)).unwrap() +} + +/// [`kll_over_last_5m`] over the group of `expression`. +fn kll_over_last_5m_of(memo: &mut Memo, expression: Operator) -> MemoGroupId { + let input = memo.find_or_add_group(expression).unwrap(); + kll_over_last_5m(memo, input) +} + +/// `@ end()` and `@ start()` are lifted above the subtree they anchor: +/// the front end lowers `m[5m] @ end()` as `TimeShift{@ end()}(G2)` over +/// G2 = `m[5m]`, the group an instant query's `m[5m]` reads too. The +/// `TimeShift` pins G2 to one end of the request, so it is not lifted: it +/// is the input group of the state over it, whose selection is empty. +/// +/// ```text +/// G1 Scan latency; G2 TimeRange[5m](G1); G3 SummaryAgg[KLL](G2) (instant query) +/// G4 TimeShift(@ end())(G2); G5 SummaryAgg[KLL](G4) (range query) +/// ``` +#[test] +fn at_end_and_at_start_share_the_subtree_they_anchor_and_stay_unlifted() { + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let g3 = memo.find_or_add_group(kll(g2)).unwrap(); + assert_eq!(time_of(&memo, g3), relative(-5 * MINUTE_MS, 0)); + for at in [AtModifier::End, AtModifier::Start] { + let pinned = memo.find_or_add_group(shift(0, Some(at), g2)).unwrap(); + assert_eq!( + memo.groups()[pinned.0 as usize].expression.children(), + [&g2] + ); + let state = memo.find_or_add_group(kll(pinned)).unwrap(); + let coverage = coverage(&memo, state); + assert_eq!(coverage.definition().input, Some(pinned)); + assert_eq!(coverage.definition().unlifted_operators, [kll_unlifted()]); + assert!(coverage.selection()[0].columns().is_empty()); + } +} + +/// `ts >= ` (a SQL timestamp filter) and `value > 10` lift as +/// intervals between fixed values: `G3 SummaryAgg[KLL](G2)`, `G2 +/// Filter(ts >= 2026-11-01 ∧ value > 10)(G1)`. +#[test] +fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + let compare = |column, op, value| ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(value)), + semantics: ExprSemantics::Sql, + }; + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g2 = memo + .find_or_add_group(filter( + ScalarExpr::BoolAnd(vec![ + compare(TS, CompareOpKind::Ge, ScalarValue::Timestamp(NOV_1_2026_MS)), + compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), + ]), + g1, + )) + .unwrap(); + let g3 = memo.find_or_add_group(kll(g2)).unwrap(); + let fixed = |value| Bound::Included(IntervalConstraintLimit::FixedValue(value)); + assert_eq!( + time_of(&memo, g3), + Constraint::Interval { + lower: fixed(ScalarValue::Timestamp(NOV_1_2026_MS)), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).selection()[0].columns()[&ColumnIdentity::new(None, "value")], + Constraint::Interval { + lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( + 10.0 + ))), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).definition().unlifted_operators, + vec![kll_unlifted()] + ); +} + +/// Counter states (`Increase`) merge only over windows disjoint in time: +/// the last 5 minutes and the 5 before them merge into the last 10; two +/// copies of the last 5 minutes do not. A KLL over the last minute and one +/// over the minute before it are disjoint too. +/// +/// ```text +/// G2 TimeRange[5m](G1); G3 SummaryAgg[Increase](G2) +/// G4 TimeShift(5m)(G1); G5 TimeRange[5m](G4); G6 SummaryAgg[Increase](G5) +/// G7 SummaryMerge(G3, G6) +/// ``` +#[test] +fn counter_states_merge_over_windows_disjoint_in_time() { + let increase = || FieldDataType::exact(ExactKind::Increase); + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let g3 = memo.find_or_add_group(state(increase(), g2)).unwrap(); + let g4 = memo + .find_or_add_group(shift(5 * MINUTE_MS, None, g1)) + .unwrap(); + let g5 = memo.find_or_add_group(range(5 * MINUTE_MS, g4)).unwrap(); + let g6 = memo.find_or_add_group(state(increase(), g5)).unwrap(); + let g7 = memo.find_or_add_group(merge(vec![g3, g6])).unwrap(); + assert_eq!(time_of(&memo, g7), relative(-10 * MINUTE_MS, 0)); + assert!(matches!( + memo.find_or_add_group(merge(vec![g3, g3])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::DisjointInTime + } + )) + )); + let g8 = memo.find_or_add_group(kll(g2)).unwrap(); + let g9 = memo.find_or_add_group(kll(g5)).unwrap(); + let g10 = memo.find_or_add_group(merge(vec![g8, g9])).unwrap(); + assert_eq!(time_of(&memo, g10), relative(-10 * MINUTE_MS, 0)); +} + +/// An exact `Aggregate` gets coverage like a summary state: `G4 +/// Aggregate[sum(value) by job](G3)` over the last minute of `job = "api"` +/// has the `Aggregate` as its definition and the job and minute as its +/// selection. +#[test] +fn an_exact_aggregate_has_coverage() { + let sum_by_job = |child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![JOB]), + measures: vec![AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }; + let mut memo = Memo::new(&DataWorkload::default()); + let g1 = memo.find_or_add_group(latency(vec![])).unwrap(); + let g2 = memo.find_or_add_group(filter(job_is("api"), g1)).unwrap(); + let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(sum_by_job(g3)).unwrap(); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input, Some(g1)); + assert_eq!( + coverage.definition().unlifted_operators, + vec![sum_by_job(MemoGroupId(0)).map_children(|_| None)] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert_eq!(selection.columns()[&ts()], last_minute()); +} + +/// KLL over the last minute of `metric{job="api"}` (`scan` → `filter` → +/// `range` → `kll`), in `memo`: its scan group and its state group. +fn kll_of_last_minute_of(memo: &mut Memo, metric: &str) -> (MemoGroupId, MemoGroupId) { + let scan = memo + .find_or_add_group(scan( + metric, + &[ + ("job", DataType::Utf8), + ("a", DataType::Float64), + ("b", DataType::Float64), + ], + vec![], + )) + .unwrap(); + (scan, kll_of_last_minute_where(memo, scan, job_is("api"))) +} + +/// The plan's view of a state's coverage names no memo group: the same +/// state in two memos whose `Scan` groups have different ids has one +/// `definition_id` and one selection. +#[test] +fn a_view_coverage_is_equal_across_memos_whatever_the_group_ids() { + let mut first = Memo::new(&DataWorkload::default()); + let (first_scan, first_state) = kll_of_last_minute_of(&mut first, "latency"); + let mut second = Memo::new(&DataWorkload::default()); + // An unrelated query first, so `latency`'s scan gets another id. + kll_of_last_minute_of(&mut second, "requests"); + let (second_scan, second_state) = kll_of_last_minute_of(&mut second, "latency"); + assert_ne!(first_scan, second_scan); + assert_eq!( + coverage(&first, first_state).definition().input, + Some(first_scan) + ); + let first_view = view_coverage(&first, first_state).unwrap(); + let second_view = view_coverage(&second, second_state).unwrap(); + assert_eq!(first_view.definition_id(), second_view.definition_id()); + assert_eq!(first_view.selection(), second_view.selection()); +} + +/// The same shape over another source computes something else: another +/// `definition_id`. +#[test] +fn a_view_coverage_over_another_source_has_another_definition_id() { + let mut memo = Memo::new(&DataWorkload::default()); + let (_, over_latency) = kll_of_last_minute_of(&mut memo, "latency"); + let (_, over_requests) = kll_of_last_minute_of(&mut memo, "requests"); + assert_ne!( + view_coverage(&memo, over_latency).unwrap().definition_id(), + view_coverage(&memo, over_requests).unwrap().definition_id() + ); +} + +/// A group without aggregate coverage (a scan) has no view coverage. +#[test] +fn a_group_without_aggregate_coverage_has_no_view_coverage() { + let mut memo = Memo::new(&DataWorkload::default()); + let (scan, _) = kll_of_last_minute_of(&mut memo, "latency"); + assert_eq!(view_coverage(&memo, scan), None); +} diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs new file mode 100644 index 000000000..005db69a0 --- /dev/null +++ b/crates/types/tests/memo_group_properties.rs @@ -0,0 +1,774 @@ +//! `column_signs`, `depends_on_evaluation_time` and `scanned_sources`, +//! derived once per memo group from its input groups (concepts doc, section +//! 3, "Value sign" and `GroupProperties`; `aggregate_coverage` is in +//! `memo_aggregate_coverage.rs`). Every example reads one of three sources: +//! +//! ```text +//! latency [ts, value, host] value declared non-negative +//! requests_total [ts, value, host] a counter: value non-negative +//! gauge [ts, value, host] nothing declared +//! ``` +use std::collections::{BTreeMap, BTreeSet}; + +use asap_types::ir::operator::arguments::{ + AtModifier, BinaryOpKind, GroupKeys, JoinKind, Reduction, Source, +}; +use asap_types::ir::operator::AggIntent; + +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::{ColumnRef, ProjectItem}; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, + ScalarExpr, +}; +use asap_types::stages::{Memo, MemoGroupId, ValueSign}; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic}; +use asap_types::test_support::memo_fixtures::{ + aggregate, filter, range, readout, scan, shift, state, +}; +use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; + +use ValueSign::{NonNegative, NonPositive, Unknown}; + +const TS: usize = 0; +const VALUE: usize = 1; +const HOST: usize = 2; +const MINUTE_MS: i64 = 60_000; + +/// The memo of every example: `latency.value` declared non-negative, and +/// `requests_total` a declared counter, whose `value` is non-negative. +fn memo() -> Memo { + Memo::new(&DataWorkload { + sources: BTreeMap::from([ + ( + "latency".to_string(), + SourceFacts { + column_signs: BTreeMap::from([("value".to_string(), NonNegative)]), + ..SourceFacts::default() + }, + ), + ( + "requests_total".to_string(), + SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + ..SourceFacts::default() + }, + ), + ]), + ..DataWorkload::default() + }) +} + +fn add(memo: &mut Memo, expression: Operator) -> MemoGroupId { + memo.find_or_add_group(expression).unwrap() +} + +fn signs(memo: &Memo, group: MemoGroupId) -> &[ValueSign] { + &memo.groups()[group.0 as usize] + .group_properties + .column_signs +} + +fn depends(memo: &Memo, group: MemoGroupId) -> bool { + memo.groups()[group.0 as usize] + .group_properties + .depends_on_evaluation_time +} + +/// `Scan metric`: `[ts, value, host]`. +fn series(metric: &str) -> Operator { + scan(metric, &[("host", DataType::Utf8)], vec![]) +} + +fn compare( + left: ScalarExpr, + op: CompareOpKind, + right: ScalarExpr, +) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(left), + op, + right: Box::new(right), + semantics: ExprSemantics::Sql, + } +} + +/// `Project(ts, expr AS value, host)`. +fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator { + let item = |expr, alias: &str| ProjectItem { + alias: Some(alias.into()), + expr, + }; + Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + item(ScalarExpr::Column(TS), "ts"), + item(expr, "value"), + item(ScalarExpr::Column(HOST), "host"), + ], + qualifier: None, + child, + }) +} + +fn by_host() -> Reduction { + Reduction::by(vec![HOST]) +} + +/// Rows of the per-operator table that keep or set a column's sign: +/// +/// | Group | Expression | `column_signs` | +/// |---|---|---| +/// | G1 | `Scan latency` | `[Unknown, NonNegative, Unknown]` (declared) | +/// | G2 | `Scan gauge` | `[Unknown, Unknown, Unknown]` (nothing declared) | +/// | G3 | `Filter(value > 1)(G1)` | G1's | +/// | G4 | `TimeRange[1m](G3)` | G1's | +/// | G5 | `TimeShift(1m)(G4)` | G1's | +/// | G6 | `Project(abs(value))(G2)` | value `NonNegative` | +/// | G7 | `Project(-value)(G1)` | value `NonPositive` (flipped) | +/// | G8 | `Project(-value)(G7)` | value `NonNegative` (flipped back) | +/// | G9 | `Project(value * 2)(G1)` | value `NonNegative` (same signs multiply) | +/// | G10 | `Project(value - value)(G1)` | value `Unknown` (a difference) | +#[test] +fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, series("gauge")); + let g3 = add( + &mut memo, + filter( + compare( + ScalarExpr::Column(VALUE), + CompareOpKind::Gt, + ScalarExpr::Literal(ScalarValue::Float64(1.0)), + ), + g1, + ), + ); + let g4 = add(&mut memo, range(MINUTE_MS, g3)); + let g5 = add(&mut memo, shift(MINUTE_MS, None, g4)); + let value = || Box::new(ScalarExpr::Column(VALUE)); + let negated = || ScalarExpr::Negative { + expr: value(), + semantics: ExprSemantics::Promql, + }; + let g6 = add( + &mut memo, + map_value( + ScalarExpr::FunctionCall { + name: "promql_abs".into(), + args: vec![ScalarExpr::Column(VALUE)], + }, + g2, + ), + ); + let g7 = add(&mut memo, map_value(negated(), g1)); + let g8 = add(&mut memo, map_value(negated(), g7)); + let g9 = add( + &mut memo, + map_value( + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: value(), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }, + g1, + ), + ); + let g10 = add( + &mut memo, + map_value( + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Sub, + left: value(), + right: value(), + semantics: ExprSemantics::Promql, + }, + g1, + ), + ); + let declared = [Unknown, NonNegative, Unknown]; + assert_eq!(signs(&memo, g1), declared); + assert_eq!(signs(&memo, g2), [Unknown; 3]); + for group in [g3, g4, g5] { + assert_eq!(signs(&memo, group), declared, "{group:?}"); + } + for (group, value) in [ + (g6, NonNegative), + (g7, NonPositive), + (g8, NonNegative), + (g9, NonNegative), + (g10, Unknown), + ] { + assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{group:?}"); + } +} + +/// Aggregates: keys keep their sign, sum / max keep the value's, counts are +/// non-negative, and a rate is non-negative over a non-negative value only: +/// +/// | Group | Expression | `column_signs` | +/// |---|---|---| +/// | G2 | `sum by (host)(G1)`, G1 = `Scan latency` | `[Unknown, NonNegative]` | +/// | G3 | `max by (host)(G1)` | `[Unknown, NonNegative]` | +/// | G5 | `sum by (host)(G4)`, G4 = `Scan gauge` | `[Unknown, Unknown]` | +/// | G6 | `count by (host)(G4)` | `[Unknown, NonNegative]` | +/// | G7 | `count(distinct value) by (host)(G4)` | `[Unknown, NonNegative]` | +/// | G9 | `rate(G8)`, G8 = `requests_total[1m]` | value `NonNegative` | +/// | G11 | `rate(G10)`, G10 = `gauge[1m]` | value `Unknown` | +#[test] +fn aggregates_keep_their_keys_and_derive_their_values() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let sum = || AggIntent::Sum { col: Some(VALUE) }; + let g2 = add(&mut memo, aggregate(by_host(), sum(), g1)); + let g3 = add( + &mut memo, + aggregate(by_host(), AggIntent::Max { col: Some(VALUE) }, g1), + ); + let g4 = add(&mut memo, series("gauge")); + let g5 = add(&mut memo, aggregate(by_host(), sum(), g4)); + let g6 = add(&mut memo, aggregate(by_host(), AggIntent::Count, g4)); + let g7 = add( + &mut memo, + aggregate(by_host(), AggIntent::Cardinality { cols: vec![VALUE] }, g4), + ); + let requests = add(&mut memo, series("requests_total")); + let g8 = add(&mut memo, range(MINUTE_MS, requests)); + let g9 = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, g8), + ); + let g10 = add(&mut memo, range(MINUTE_MS, g4)); + let g11 = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, g10), + ); + for (group, value) in [ + (g2, NonNegative), + (g3, NonNegative), + (g5, Unknown), + (g6, NonNegative), + (g7, NonNegative), + ] { + assert_eq!(signs(&memo, group), [Unknown, value], "{group:?}"); + } + assert_eq!(signs(&memo, g9), [Unknown, NonNegative, Unknown]); + assert_eq!(signs(&memo, g11), [Unknown; 3]); +} + +/// A state records the sign of the values it is updated with, a merge +/// keeps the sign its inputs agree on, and a readout reads the state's, a +/// count-like statistic being non-negative whatever the values: +/// +/// | Group | Expression | `column_signs` | +/// |---|---|---| +/// | G3 | `SummaryAgg[KLL](G2)`, G2 = `latency[1m]` | state `NonNegative` | +/// | G6 | `SummaryAgg[KLL](G5)`, G5 = `(latency offset 1m)[1m]` | state `NonNegative` | +/// | G7 | `SummaryMerge(G3, G6)` | state `NonNegative` (agreed) | +/// | G8 | `SummaryEstimate[p99](G7)` | quantile `NonNegative` | +/// | G11 | `SummaryAgg[HLL](G10)`, G10 = `gauge[1m]` | state `Unknown` | +/// | G12 | `SummaryEstimate[cardinality](G11)` | cardinality `NonNegative` | +#[test] +fn states_record_their_input_sign_and_readouts_read_it() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, range(MINUTE_MS, g1)); + let g3 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g2), + ); + let g4 = add(&mut memo, shift(MINUTE_MS, None, g1)); + let g5 = add(&mut memo, range(MINUTE_MS, g4)); + let g6 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g5), + ); + let g7 = add( + &mut memo, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![g3, g6], + group_by: None, + }), + ); + let g8 = add( + &mut memo, + readout(SketchStatistic::Quantile { q: 0.99 }, g7), + ); + let g9 = add(&mut memo, series("gauge")); + let g10 = add(&mut memo, range(MINUTE_MS, g9)); + let g11 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Hll), g10), + ); + let g12 = add(&mut memo, readout(SketchStatistic::Cardinality, g11)); + // [ts, state, host] + for group in [g3, g6, g7] { + assert_eq!( + signs(&memo, group), + [Unknown, NonNegative, Unknown], + "{group:?}" + ); + } + assert_eq!(signs(&memo, g8), [Unknown, NonNegative, Unknown]); + assert_eq!(signs(&memo, g11), [Unknown; 3]); + assert_eq!(signs(&memo, g12), [Unknown, NonNegative, Unknown]); +} + +/// The concepts doc's example: `topk(5, sum by (host)(gauge))` ranks values +/// of `Unknown` sign (Stage 2 then prunes the Count-Min heap builds), while +/// `topk(5, sum by (host)(rate(requests_total[1m])))` ranks `NonNegative` +/// ones. +#[test] +fn a_top_k_ranks_values_of_its_inputs_sign() { + let mut memo = memo(); + let top_5 = |memo: &mut Memo, child| { + add( + memo, + aggregate( + Reduction::Reduce(GroupKeys::by(vec![])), + AggIntent::TopK { k: 5 }, + child, + ), + ) + }; + let sum = AggIntent::Sum { col: Some(VALUE) }; + let gauge = add(&mut memo, series("gauge")); + let gauge_sum = add(&mut memo, aggregate(by_host(), sum.clone(), gauge)); + let gauge_top = top_5(&mut memo, gauge_sum); + let requests = add(&mut memo, series("requests_total")); + let ranged = add(&mut memo, range(MINUTE_MS, requests)); + let rate = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, ranged), + ); + let rate_sum = add(&mut memo, aggregate(by_host(), sum, rate)); + let rate_top = top_5(&mut memo, rate_sum); + // [host, sum]: the ranked value is column 1 of the ranked rows. + assert_eq!(signs(&memo, gauge_sum), [Unknown, Unknown]); + assert_eq!(signs(&memo, rate_sum), [Unknown, NonNegative]); + let ranked = |group| { + let schema = &memo.groups()[group as usize].group_properties.schema; + let value = schema.column_id("topk_5").unwrap(); + signs(&memo, MemoGroupId(group))[value] + }; + assert_eq!(ranked(gauge_top.0), Unknown); + assert_eq!(ranked(rate_top.0), NonNegative); +} + +/// `depends_on_evaluation_time`: a `TimeRange` reads the window before the +/// evaluation time, `now()` reads it directly, and everything above them +/// depends on it, unless `@ T` pins the subtree; `@ end()` does not pin it +/// to a fixed time. +/// +/// | Group | Expression | Depends | +/// |---|---|---| +/// | G1 | `Scan latency` | no | +/// | G2 | `sum by (host)(G1)` | no | +/// | G3 | `latency[1m]` | yes | +/// | G4 | `SummaryAgg[KLL](G3)` | yes | +/// | G5 | `TimeShift(@ T, offset 1m)(G3)` | no | +/// | G6 | `SummaryAgg[KLL](G5)` | no | +/// | G7 | `TimeShift(@ end(), offset 1m)(G3)` | yes | +/// | G8 | `Filter(ts > now())(G1)` | yes | +#[test] +fn a_result_depends_on_the_evaluation_time_under_a_time_range_or_now() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add( + &mut memo, + aggregate(by_host(), AggIntent::Sum { col: Some(VALUE) }, g1), + ); + let g3 = add(&mut memo, range(MINUTE_MS, g1)); + let g4 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g3), + ); + let g5 = add( + &mut memo, + shift( + MINUTE_MS, + Some(AtModifier::Timestamp(1_700_000_000_000)), + g3, + ), + ); + let g6 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g5), + ); + let g7 = add(&mut memo, shift(MINUTE_MS, Some(AtModifier::End), g3)); + let g8 = add( + &mut memo, + filter( + compare( + ScalarExpr::Column(TS), + CompareOpKind::Gt, + ScalarExpr::CurrentTimestamp, + ), + g1, + ), + ); + for (group, expected) in [ + (g1, false), + (g2, false), + (g3, true), + (g4, true), + (g5, false), + (g6, false), + (g7, true), + (g8, true), + ] { + assert_eq!(depends(&memo, group), expected, "{group:?}"); + } +} + +/// `scanned_sources`, derived once per group: a `Scan` its source, any other +/// group the union over its input groups. A PromQL `Scan` of a time series +/// and a `TimeRange` over it scan only time series +/// (`Source::is_time_series`); a SQL table `Scan` does not, nor a `Join` +/// reading two (a SQL join reads relations only, so a time series cannot be +/// one of its inputs). +#[test] +fn a_group_scans_the_sources_of_the_scans_below_it() { + let mut memo = memo(); + let series = add(&mut memo, series("latency")); + let window = add(&mut memo, range(MINUTE_MS, series)); + let table = |name: &str| { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: name.into(), + }, + predicates: vec![], + schema: Schema::new(vec![ + Field::plain("host", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + }) + }; + let requests = add(&mut memo, table("requests")); + let hosts = add(&mut memo, table("hosts")); + let join = add( + &mut memo, + Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: requests, + right: hosts, + }), + ); + let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; + let table_source = |name: &str| Source::Table { + table_ref: name.into(), + }; + assert_eq!( + properties(join).scanned_sources, + BTreeSet::from([table_source("hosts"), table_source("requests")]) + ); + assert_eq!( + properties(window).scanned_sources, + BTreeSet::from([Source::TimeSeries { + metric: "latency".into() + }]) + ); + let reads = |group: MemoGroupId| { + properties(group) + .scanned_sources + .iter() + .all(Source::is_time_series) + }; + assert!(reads(series)); + assert!(reads(window)); + assert!(!reads(requests)); + assert!(!reads(join)); +} + +/// `count_values without (host) ("version", latency)`: a `without` +/// aggregate's layout adds no label column for `count_values` +/// (`aggregate_output_schema`), so the group has one sign per output +/// column, the count's non-negative. +#[test] +fn count_values_without_has_one_sign_per_output_column() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add( + &mut memo, + aggregate( + Reduction::Reduce(GroupKeys::without(vec![HOST])), + AggIntent::CountValues { + label: "version".into(), + }, + g1, + ), + ); + let group = &memo.groups()[g2.0 as usize].group_properties; + assert_eq!(group.column_signs.len(), group.schema.fields.len()); + assert_eq!(group.column_signs.last(), Some(&NonNegative)); +} + +/// A group's properties come from its operator and its input groups' +/// properties: the scan's schema, the state's kind and state column, the +/// readout's value column, and whether equal copies may be one group. +#[test] +fn group_properties_come_from_input_groups() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let api = compare( + ScalarExpr::Column(HOST), + CompareOpKind::Eq, + ScalarExpr::Literal(ScalarValue::Utf8("api".into())), + ); + let g2 = add(&mut memo, filter(api, g1)); + let g3 = add(&mut memo, range(MINUTE_MS, g2)); + let g4 = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g3), + ); + let readout = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Quantile { q: 0.99 }, + })) + .unwrap(); + let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; + assert_eq!(properties(g1).schema.fields.len(), 3); + assert_eq!(properties(g1).aggregate_coverage, None); + assert_eq!(properties(g4).result_kind, OperatorResultKind::State); + assert!(properties(g4) + .schema + .fields + .iter() + .any(|field| field.name == "state" && !field.is_plain())); + assert!(properties(g4).aggregate_coverage.is_some()); + assert_eq!( + properties(readout).result_kind, + OperatorResultKind::InstantVector + ); + assert!(properties(readout) + .schema + .fields + .iter() + .any(|field| field.name == "quantile" + && field.dtype == FieldDataType::Plain(DataType::Float64))); + assert!(properties(g4).shareable_when_equal); +} + +/// A point count read from a Count Sketch is a median of signed counters, +/// which can be negative over non-negative weights: `Unknown`; one read +/// from a Count-Min state, an overestimate of a sum of non-negative +/// weights, keeps the weights' sign. +#[test] +fn a_count_sketch_point_count_has_no_sign_a_count_min_one_keeps_it() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, range(MINUTE_MS, g1)); + let point_count = |memo: &mut Memo, algorithm| { + let state = add(memo, state(FieldDataType::Sketch(algorithm), g2)); + add( + memo, + readout( + SketchStatistic::PointCount { + key: ColumnRef::Named("host".into()), + value: Some("a".into()), + }, + state, + ), + ) + }; + let count_sketch = point_count(&mut memo, SketchAlgorithm::CountSketch); + let count_min = point_count(&mut memo, SketchAlgorithm::Cms); + // [ts, count, host]: the count is the state's column. + let value = |group: MemoGroupId| { + let schema = &memo.groups()[group.0 as usize].group_properties.schema; + signs(&memo, group)[schema.column_id("count").unwrap()] + }; + assert_eq!(value(count_sketch), Unknown); + assert_eq!(value(count_min), NonNegative); +} + +/// PromQL's `quantile(φ, v)` with φ outside [0, 1] is ±Inf, so a quantile +/// keeps its input's sign only for 0 ≤ φ ≤ 1, as a measure and as a +/// sketch readout. +#[test] +fn a_quantile_outside_the_unit_interval_has_no_sign() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let quantile = |memo: &mut Memo, q| { + add( + memo, + aggregate( + by_host(), + AggIntent::Quantile { + col: Some(VALUE), + q, + }, + g1, + ), + ) + }; + let inside = quantile(&mut memo, 0.5); + let below = quantile(&mut memo, -0.5); + let g2 = add(&mut memo, range(MINUTE_MS, g1)); + let kll = add( + &mut memo, + state(FieldDataType::Sketch(SketchAlgorithm::Kll), g2), + ); + let read = add( + &mut memo, + readout(SketchStatistic::Quantile { q: 1.5 }, kll), + ); + let value = |group: MemoGroupId| *signs(&memo, group).last().unwrap(); + assert_eq!(value(inside), NonNegative); + assert_eq!(value(below), Unknown); + assert_eq!(value(read), Unknown); +} + +fn arithmetic(op: ArithmeticOpKind, lhs: MemoGroupId, rhs: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(op), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) +} + +/// Arithmetic between two vectors: a sum of non-negatives, and a product or +/// quotient of same-signed values, is non-negative; a difference has no +/// sign. A `vector(s)` value has `s`'s sign: +/// +/// | Group | Expression | value sign | +/// |---|---|---| +/// | G3 | `quantile_over_time(0.9, G2)`, G2 = `latency[1h]` | `NonNegative` | +/// | G4 | `quantile_over_time(0.5, G2)` | `NonNegative` | +/// | G5 | `G3 / G4` | `NonNegative` | +/// | G6 | `G3 + G4` | `NonNegative` | +/// | G7 | `G3 - G4` | `Unknown` | +/// | G8 | `vector(1000)` | `NonNegative` | +/// | G9 | `vector(-1)` | `NonPositive` | +/// | G10 | `G3 * G8` | `NonNegative` | +/// | G11 | `G3 * G9` | `NonPositive` | +/// | G12 | `G9 * G9` | `NonNegative` | +#[test] +fn vector_arithmetic_follows_its_operands_signs() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let g2 = add(&mut memo, range(60 * MINUTE_MS, g1)); + let quantile = |q| AggIntent::Quantile { + col: Some(VALUE), + q, + }; + let g3 = add( + &mut memo, + aggregate(Reduction::PerEntity, quantile(0.9), g2), + ); + let g4 = add( + &mut memo, + aggregate(Reduction::PerEntity, quantile(0.5), g2), + ); + let g5 = add(&mut memo, arithmetic(ArithmeticOpKind::Div, g3, g4)); + let g6 = add(&mut memo, arithmetic(ArithmeticOpKind::Add, g3, g4)); + let g7 = add(&mut memo, arithmetic(ArithmeticOpKind::Sub, g3, g4)); + let vector = |s: f64| { + Operator::NonASAP(NonASAPOp::PromqlVectorFromScalar(ScalarExpr::Literal( + ScalarValue::Float64(s), + ))) + }; + let g8 = add(&mut memo, vector(1000.0)); + let g9 = add(&mut memo, vector(-1.0)); + let g10 = add(&mut memo, arithmetic(ArithmeticOpKind::Mul, g3, g8)); + let g11 = add(&mut memo, arithmetic(ArithmeticOpKind::Mul, g3, g9)); + let g12 = add(&mut memo, arithmetic(ArithmeticOpKind::Mul, g9, g9)); + let value = |memo: &Memo, group: MemoGroupId| { + let schema = &memo.groups()[group.0 as usize].group_properties.schema; + signs(memo, group)[schema.column_id("value").unwrap()] + }; + for (group, sign) in [ + (g3, NonNegative), + (g4, NonNegative), + (g5, NonNegative), + (g6, NonNegative), + (g7, Unknown), + (g8, NonNegative), + (g9, NonPositive), + (g10, NonNegative), + (g11, NonPositive), + (g12, NonNegative), + ] { + assert_eq!(value(&memo, group), sign, "{group:?}"); + } +} + +/// A vector-scalar operation lowers to a `Project` of an arithmetic +/// expression (PromQL `x / 1000`, `x * 8`), as does SQL's `value * 8`; the +/// expression's sign follows the same rule as vector arithmetic: +/// +/// | Group | Expression | value sign | +/// |---|---|---| +/// | G2 | `G1 / 1000`, G1 = `Scan latency` (PromQL) | `NonNegative` | +/// | G3 | `G1 * 8` (PromQL) | `NonNegative` | +/// | G4 | `SELECT value * 8 FROM G1` (SQL) | `NonNegative` | +/// | G5 | `G1 - G1` | `Unknown` | +/// | G6 | `G1 * -8` | `NonPositive` | +/// | G7 | `G1 / -0` | `NonPositive` (`-0` has its sign bit set: the quotient is `-inf`, `-0` or NaN) | +#[test] +fn scalar_arithmetic_follows_its_operands_signs() { + let mut memo = memo(); + let g1 = add(&mut memo, series("latency")); + let expr = |op, right: ScalarExpr, semantics| ScalarExpr::Arithmetic { + op, + left: Box::new(ScalarExpr::Column(VALUE)), + right: Box::new(right), + semantics, + }; + let number = |n: f64| ScalarExpr::Literal(ScalarValue::Float64(n)); + let promql = ExprSemantics::Promql; + let g2 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Div, number(1000.0), promql), g1), + ); + let g3 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Mul, number(8.0), promql), g1), + ); + let g4 = add( + &mut memo, + map_value( + expr( + ArithmeticOpKind::Mul, + ScalarExpr::Literal(ScalarValue::Int64(8)), + ExprSemantics::Sql, + ), + g1, + ), + ); + let g5 = add( + &mut memo, + map_value( + expr(ArithmeticOpKind::Sub, ScalarExpr::Column(VALUE), promql), + g1, + ), + ); + let g6 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Mul, number(-8.0), promql), g1), + ); + let g7 = add( + &mut memo, + map_value(expr(ArithmeticOpKind::Div, number(-0.0), promql), g1), + ); + for (group, value) in [ + (g2, NonNegative), + (g3, NonNegative), + (g4, NonNegative), + (g5, Unknown), + (g6, NonPositive), + (g7, NonPositive), + ] { + assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{group:?}"); + } +} diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 828ee1993..23d9e93a8 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -298,6 +298,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }) .collect(); let inputs: Vec<_> = (0..count).chain([0, 0]).collect(); @@ -313,6 +314,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { guarantee: None, retention_ms: None, selected_physical_candidate: None, + view_coverage: None, }); let mut dag = PhysicalASAPDAG { nodes, diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index b24531056..84cda9c5a 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -216,7 +216,7 @@ struct LabelSetDistinctValues { | `rows_per_second` | With `Ingesting`/`Mixed` | Rows arriving per second over all sources, used to price scans and continuous maintenance. It must be finite and non-negative. A source with no `series_count` + `sample_interval` of its own is priced at its series share of it when every source declares `series_count`, else at all of it. | | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | | `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | -| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan`'s `column_sign` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | +| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan` group's `column_signs` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | | `distinct_values_per_label` | Optional; default empty; each count nonzero | Distinct values each label (PromQL) or column (SQL) takes, e.g. `{"host": 1000, "job": 10}`. Plan selection prices a `by (k1, k2, ...)` aggregate or summary, and a partitioned sort or limit, at the product of its keys' counts, at most the `series_count` (series, or groups of rows) of the sources it reads and the rows it reads per evaluation. Over several sources a key's counts add. No count is assumed for a key some source read does not declare: the grouping is priced at the smaller of that `series_count` and those rows. Each count must be an upper bound, counting a missing label (or a NULL) as one more value: when every key of a grouped result declares one, their product (at most the series read) also bounds `G`, which sizes per-group sketches for a whole-result target; otherwise `G` keeps its series bound, or none for SQL. | diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 3eb03b8c5..597811fd6 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -540,7 +540,7 @@ pub struct MaterializedViewRequest<'a> { | | `read_raw_data` | `read_materialized_view` | | --- | --- | --- | -| Keyed by | `Source` | `MaterializedView` (its coverage: `definition_id` and selection) | +| Keyed by | `Source` | `MaterializedView` (its coverage: the content id `definition_id` and selection) | | Time range | optional, a hint | always given, exact | | Predicates | a hint | none | | Executor re-filters | yes | no | @@ -663,7 +663,12 @@ Query at T = 12:05 over (12:00, 12:05]: ``` The deployment matches a view to stored states by its coverage -(`definition_id` and selection). At execution time it checks their actual +(`definition_id` and selection). Identity has two scopes: the memo compares +coverages by `AggregateDefinition` `==` (its definitions hold memo group ids) +and needs no id; the plan carries the view's content id on its node +(`PhysicalASAPDAGNode::view_coverage`), defined at +[`ViewCoverage::definition_id`](../../crates/types/src/plan/physical_asap_dag.rs). +At execution time it checks their actual coverage, revisions and readiness. A compiled plan cannot establish future readiness. The deployment does not replace `NativeKllMerge`, choose another sketch, or maintain different windows: such changes require replanning. @@ -682,7 +687,9 @@ sketch, or maintain different windows: such changes require replanning. | Where does retention live? | On the materialized-view node of the exported DAG | — | Mapping the materialized-views extension's terms: its `query()` is the sub-DAG -under the view's node, and `coverage.definition_id` identifies that sub-DAG. +under the view's node, and `coverage.definition_id` (the content id, +[`ViewCoverage::definition_id`](../../crates/types/src/plan/physical_asap_dag.rs)) +identifies it. Its readiness is the summary store's status, which the deployment owns. Its `mv_dependencies` build graph is `MaterializedView.inputs`. @@ -741,7 +748,7 @@ operator/runtime fixtures: | Test | Contract exercised | | --- | --- | | `kll_tumbling_window_execution::five_tumbling_windows_roundtrip_and_shared_merge_runs_once` | One-minute tumbling window refreshes from raw data → five materialized view slots → shared native merge → p50/p99; counts every sample once, checks adjacent aligned windows and instruments one merge start per run | -| `kll_tumbling_window_execution::a_store_keyed_by_definition_and_time_range_answers_the_tumbling_windows` | A store that finds states only by `(definition_id, time range)` answers the five-tumbling-window query: the tumbling windows share one definition and request the five ranges that tile the window | +| `kll_tumbling_window_execution::a_store_keyed_by_definition_and_time_range_answers_the_tumbling_windows` | A store that finds states only by `(definition_id, time range)` (the content id is an opaque string to the store) answers the five-tumbling-window query: the tumbling windows share one definition and request the five ranges that tile the window | | `runtime_install::tumbling_windows_of_one_refresh_window_read_each_raw_window_once` | A KLL chain and a DDSketch chain of 10 s tumbling windows over one `Scan` are one refresh group: each tumbling window window is read from raw data once, in one ingestion run that builds both tumbling windows; each state equals that of a run of its view alone, and both queries answer as the query-time reference | | `kll_tumbling_window_execution::tumbling_windows_reject_parameters_schema_and_missing_binding` | Parameter relabelling, incompatible schemas and missing views fail explicitly | | `precompute_candidates::grouped_rate_can_be_materialized_before_or_after_grouped_sum` | Stored Rate state and stored Sum state answer like query time; summing counters before Rate does not | diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index e0a25e1a3..cfa260d4c 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -114,10 +114,10 @@ A node in the physical data will represent the data or summary instance, so a no | Why not put it in the schema? | States worth merging cover different data but must have the same schema | below | | What is it based on? | Goldstein & Larson view matching (SIGMOD 2001), explained with an example | §4.1 | | What does it store? | `definition` (what is computed) + `selection` (which rows were taken) | §4.2.1 | -| How is it computed? | by the planner, from the sub-DAG the node covers, in one walk that yields both the definition and the selection | §4.2.2 | -| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; cached per node | §4.2.3 | +| How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | +| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `OperatorNode::coverage()`, `SummaryCoverage::derive` | §6.1, §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, `Operator::aggregate_coverage` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -192,7 +192,7 @@ GROUP BY job; **Existing implementation.** The `WHERE` split is implemented for DataFusion in [`datafusion-contrib/datafusion-materialized-views`](https://github.com/datafusion-contrib/datafusion-materialized-views), `src/rewrite/normal_form.rs` (`SpjNormalForm`, `Predicate { eq_classes, ranges_by_equivalence_class, residuals }`). It rejects plans that contain an `Aggregate` or a `Join`. -### 4.2 Summary Coverage = Summary definition + selection +### 4.2 Aggregate coverage = aggregate definition + selection #### 4.2.1 Why coverage has two parts @@ -237,70 +237,121 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -The planner computes the coverage of a node from the sub-DAG the node covers, not from a declaration. One walk down the sub-DAG produces both parts: every filter condition either moves into `selection` or stays in `definition`. The definition is described first, then how the planner decides which conditions move into the selection. +Coverage is an IR property, derived like an output schema and never declared: `Operator::aggregate_coverage` (`crates/types/src/ir/properties/aggregate_coverage.rs`) walks down from a `SummaryAgg` or `Aggregate` through its inputs' operators, generic over how a definition names an input (`OperatorInput::Reference`). The memo derives it once, when `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), over the input groups' `expression`s (`R` = `MemoGroupId`), and stores it in the group's `GroupProperties::aggregate_coverage`; node trees store none. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). -**Worked example.** p99 latency per job over table `t` (columns `job`, `region`, `latency`): +```rust +/// What a summary state computes, with every lifted selection removed; +/// `R` names an input (a memo group's id in the memo). +pub struct AggregateDefinition { + /// The input the unlifted operators read: the first input under the + /// SummaryAgg or Aggregate that is not Filter / range TimeRange / + /// TimeShift / direct-column Project. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator. + pub input: Option, + /// The operators not lifted into the selection, listed top down from the + /// SummaryAgg, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or `input`); + /// `Some(r)` is another input it reads, such as the input a scalar + /// subquery in a Filter reads. + pub unlifted_operators: Vec>>, +} -```sql -SELECT job, - approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) -FROM (SELECT job, region AS r, latency - FROM t - WHERE region = 'us' AND latency * 2 > 10) -GROUP BY job; +pub struct AggregateCoverage { + pub(crate) definition: AggregateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. + pub(crate) selection: Vec, +} + +impl AggregateCoverage { + /// The same coverage with each input reference `r` replaced by `f(r)`. + pub fn map_references(self, f: impl FnMut(R) -> S) -> AggregateCoverage; +} + +impl Operator { + pub fn aggregate_coverage(&self) -> Result, CoverageError>; +} ``` -The SQL frontend lowers it as follows: `FILTER (WHERE …)` becomes the aggregate's filter, `region AS r` becomes a `Project`, and the `WHERE` conditions are folded into the `Scan`'s predicates. With an error target, the planner answers the quantile from a KLL `SummaryAgg`, read out by a `SummaryEstimate`. The sub-DAG under that `SummaryEstimate`: +**Worked example.** Three queries over the series `latency` (labels `job`, `a`, `b`), each a KLL state over the last minute: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job, filter: latency < 100 - ▲ - │ - [ Project ] job, region AS r, latency - ▲ - │ - [ Scan t ] predicates: region = 'us' AND latency * 2 > 10 +G1 Scan latency +G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) ← Q1 +G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) ← Q2 +G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) ← Q3 ``` -##### The definition +**Case 1, G4: everything is lifted.** The walk starts at G4's `SummaryAgg` and reads each input group's `expression`: +| Step | Group | Expression | What the walk does | +|---|---|---|---| +| 1 | G4 | `SummaryAgg[KLL](G3)` | the first unlifted operator, with no filter of its own | +| 2 | G3 | `TimeRange[1m](G2)` | the one range window: lifted as `ts ∈ (t − 60 s, t]` (t the evaluation time) | +| 3 | G2 | `Filter(job = "api")(G1)` | `job = "api"` is a value set on one column: lifted, nothing of the `Filter` remains | +| 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input` | -The `definition` is the `SummaryAgg` together with its sub-DAG, with every condition that moves into `selection` (below) taken out. Everything else stays exactly as it is: +```text +G4.aggregate_coverage = AggregateCoverage { + definition: AggregateDefinition { + input: Some(G1), + unlifted_operators: [SummaryAgg[KLL]], + }, + selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], +} +``` + +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) (`==`); its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. + +**Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: -| In the sub-DAG | In the `definition` | +```text +G10.aggregate_coverage = AggregateCoverage { + definition: AggregateDefinition { + input: Some(G1), + unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], + }, + selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], +} +``` + +Its definition differs from G4's, so G10 cannot merge with G4, even though both select `job = api`. + +**The fields.** + +- `input`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). +- `unlifted_operators`: what the state computes over `input`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. +- `selection`: which output rows of that computation the state took, as a union of boxes. +- Identity, in two scopes. In the memo, coverages compare by `AggregateDefinition` `==` (it holds `MemoGroupId`s); nothing in the memo needs an id. The plan carries the view's content id, [`ViewCoverage::definition_id`](../../../crates/types/src/plan/physical_asap_dag.rs) (defined there), on the view's plan node (`PhysicalASAPDAGNode::view_coverage`), made by `view_coverage(memo, group)`, which expands the memo's coverage (`map_references` to each input group's expression, down to the `Scan`s) rather than re-deriving it; a summary store files states under it together with a concrete selection. + +##### The definition + +The `definition` is what remains of the operators the walk passed, with every condition that moves into `selection` (below) taken out: + +| On the walk | In `unlifted_operators` | |---|---| | a `Filter` whose conditions all move into `selection` | removed | -| a `Filter` with some conditions that stay | kept, with only the conditions that stay | -| `Scan.predicates` and `SummaryAgg.filter` | trimmed the same way | -| the one range `TimeRange`, and any `TimeShift`s, when they become relative time | removed | -| a `TimeShift` with no range `TimeRange`, or two or more range `TimeRange`s | unchanged: no time is taken out | -| any other operator | unchanged | +| a `Filter` with some conditions that stay | kept, with only the conditions that stay; a scalar subquery among them names its group | +| `SummaryAgg.filter` | trimmed the same way | +| the one range `TimeRange`, and every `TimeShift` (an offset or `@ T`; not `@ start()` or `@ end()`, which stay unlifted), when they become one interval on the timestamp column | removed | +| a `TimeShift` with no range `TimeRange`, two or more range `TimeRange`s, two `@`s, or an interval that does not combine with a timestamp condition already lifted | unchanged: no time is taken out | +| a direct-column `Project` | unchanged | +| a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input` is `None` | +| any other operator (the first group the walk cannot pass) | not in the list: it is the `input` | -So the `definition` holds what the state computes: the computation `C` with its remaining conditions, the summary family and its parameters, the input column, and the grouping `G`. - -In the worked example, `latency < 100` and `region = 'us'` move into `selection` (below), and `latency * 2 > 10` stays: +An exact `Aggregate` is the first unlifted operator, kept whole (its per-measure filters and `HAVING` read its input or output, not the rows it reads). A `SummaryTumblingWindowChain[w](X)` has the definition of a build over X, and its selection leaves the timestamp column unconstrained: a chain has no range. A `TimeRange` or `TimeShift` over a chain (a group whose output is the chain's state with a time index) has the chain's coverage with the interval they select on the timestamp column of X, computed as over raw rows. A `SummaryMerge` over that group reads it, so `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` covers exactly what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does, and `Memo::add_candidate` accepts the merge as a candidate of the direct build's group: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job ← filter removed - ▲ - │ - [ Project ] job, region AS r, latency ← unchanged - ▲ - │ - [ Scan t ] predicates: latency * 2 > 10 ← region = 'us' removed +G1 Scan latency +G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) ts ∈ (t − 65m, t − 60m] +G5 Chain[KLL, 1m](G1) ts unconstrained +G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) ts ∈ (t − 65m, t − 60m] +G4 candidate: SummaryMerge(G7) the same coverage as G4 ``` -**When two definitions are equal.** Merging (§5.6) requires equal definitions. +A rollup (`SummaryMerge` with `group_by`) has its input's definition with the `SummaryAgg` grouped by the rollup's keys. -- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. -- `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. -- States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. +**When two definitions are equal.** Merging (§5.6) requires equal definitions: the same `input` and the same `unlifted_operators`, compared as values. A definition with a scalar subquery compares like any other: two states whose `Filter(a > (SELECT threshold))` read the same group are equal there. Timing is not part of a group, so a tumbling window built at ingestion time can merge with one built at query time. States over different tables never merge: a KLL over `m1` and one over `m2` read different groups. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. ##### The selection @@ -309,7 +360,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **Steps.** -1. **Collect the conditions.** Go down the sub-DAG from the `SummaryAgg` and collect every filter condition: from `Filter` nodes, from `Scan.predicates`, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. +1. **Collect the conditions.** Go down the input groups from the `SummaryAgg` and collect every filter condition: from `Filter` groups, from the `Scan.predicates` where the walk stops, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. 2. **Ask two questions about each condition:** - **Rule 1: would it pick the same rows if it were moved to just below the `SummaryAgg`?** `region = 'us'` below a `Project` that only renames columns: yes. `value > 5` below `rate`: no, because it filters the raw samples that `rate` reads, which changes the rate values. - **Rule 2: is it a simple condition on one column?** That is, a value set such as `region IN ('us', 'eu')`, or a range such as `latency < 100`. `latency * 2 > 10` is not: it is on an expression. @@ -318,7 +369,16 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` - If either answer is no: leave it in the sub-DAG, so it is part of `definition`. The paper calls such conditions *residuals*. - One more case stays in `definition`: a condition on a column that already has a lifted condition it cannot be combined with (a forbidden-value set and a range, or values of different types). Conditions are taken in walk order: the `SummaryAgg.filter`, then the `Filter`s from top to bottom, then `Scan.predicates`. -In the worked example: +A SQL example, p99 latency per job over table `t` (columns `job`, `region`, `latency`), whose `WHERE` conditions the SQL front end folds into the `Scan`'s predicates and whose `region AS r` is a `Project`: + +```sql +SELECT job, + approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) +FROM (SELECT job, region AS r, latency + FROM t + WHERE region = 'us' AND latency * 2 > 10) +GROUP BY job; +``` | Condition | Found at | Rule 1: same rows at the `SummaryAgg`? | Rule 2: simple? | Result | |---|---|---|---|---| @@ -333,11 +393,11 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl | Operator it must pass | Can it pass? | Why | Example | |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | -| `TimeRange` (range) or `TimeShift` (without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | +| `TimeRange` (range) or `TimeShift` (an offset or `@ T`; `@ start()` and `@ end()` are not lifted) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | | `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | -| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit`, a `TimeShift` with `@` | no | | | +| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit` | no | | | A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. How a PromQL comparison is lowered decides whether it is such a `Filter`: @@ -377,18 +437,21 @@ Any other shape stays in `definition`: - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. -**Time.** Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`. Its KLL state (one per series) has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = time `(−3m, −2m]`, the minute that ended 2 minutes before evaluation. There are two kinds of time: +**Time.** The timestamp column is a column like any other: a selection constrains it with an interval whose ends are `IntervalConstraintLimit`s, `EvaluationTimePlusMs(ms)` (relative to the evaluation time t) or `FixedValue(Timestamp(ms since the epoch))` (absolute). The walk turns the one range `TimeRange(W)` and the `TimeShift`s on its path, whose offsets sum to o, into `ts ∈ (anchor − o − W, anchor − o]` on the timestamp column of the `TimeRange`'s input. Like a PromQL range, which `m[r]` evaluated at t reads as (t − r, t], the interval is left-open and right-closed. The anchor is t, or T for `@ T`. `@ end()` and `@ start()` pin a subtree to one end of the request: the PromQL front end lowers the subtree X without them and puts `TimeShift { at: End | Start }(X)` above it, so X is the group of the same expression without `@`. That `TimeShift` is not lifted: the walk stops at it, and it is the definition's `input`. Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`; its KLL state has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = `ts ∈ (t − 3m, t − 2m]`. -| Kind | Comes from | Example | +| Query fragment | `lower` | `upper` | |---|---|---| -| **Relative** to the evaluation time | a range `TimeRange(w)` over a `TimeShift(s)` → `(−(s+w), −s]`; a SQL trailing window `ts >= now() - INTERVAL 'w'` lowers to `TimeRange(w)` | `TimeRange(1m)` alone → `(−1m, 0]`; over `TimeShift(1m)` → `(−2m, −1m]` | -| **Absolute** (later) | an interval on the timestamp column | `ts >= t0 AND ts < t1` → `ts ∈ [t0, t1)` | +| PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | +| PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | +| PromQL `m[5m] @ end()` or `@ start()` | the subtree `m[5m]` is the group of `m[5m]`; the `TimeShift { at: End \| Start }` above it is not lifted (it is `input`) | | +| PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | +| SQL `WHERE ts >= now() - INTERVAL '5 minutes'` | lowers to `TimeRange(5m)`: as `m[5m]` | | +| SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(1_793_491_200_000)))` | `Unbounded` | -- PromQL windows exclude their start, so relative windows are open on the left. A SQL trailing window takes the same timing, `>=` included. -- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) builds its tumbling windows this way: a 3-minute window as tumbling windows `(−1m, 0]`, `(−2m, −1m]`, `(−3m, −2m]`, so tumbling window times are derived, not declared (#601). +- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) reads a longer range from a chain: `TimeRange(3m)` over a 1-minute chain selects the windows `(t − 1m, t]`, `(t − 2m, t − 1m]`, `(t − 3m, t − 2m]`, and the merge over them covers `(t − 3m, t]`. +- Counter states (`Rate`, `Increase`) merge only when their intervals on the timestamp column are disjoint (`MergeInputRule::DisjointInTime`); other families need any column disjoint. - An instant `TimeRange` (latest sample per series) does not pick rows by time, so it stays in `definition`. The walk also stops there, so for now the label matchers of an instant selector stay in `definition` too; lifting them is planned together with `rate` (Rule 1). -- The IR cannot yet write a timestamp constant, so absolute SQL time filters stay in `definition` for now. -- Absolute and relative time are never compared: a state over `(−1m, 0]` and one over `ts ∈ [t0, t1)` are treated as possibly overlapping. +- Limits of different variants never compare: a state over `ts ∈ (t − 1m, t]` and one over `ts ∈ (t0, t1]` are treated as possibly overlapping, and a relative interval and an absolute timestamp condition on one path do not combine (time then stays in `definition`). **More examples** of what goes where. Each query's quantile is answered from a KLL `SummaryAgg`: @@ -396,7 +459,7 @@ Any other shape stays in `definition`: |---|---|---|---| | SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `region ∈ {us}` | | SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `latency ∈ (−∞, 100)` | -| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | the last 3 to 2 minutes before evaluation, `(−3m, −2m]` | +| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `ts ∈ (t − 3m, t − 2m]` | | PromQL `quantile(0.99, rate(m{job="api"}[5m]))` | `rate(TimeRange(5m, Scan m {job = 'api'}))` | all of it | nothing: `job = 'api'` is under `rate`, which the walk cannot pass yet (Rule 1, later) | | PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | `rate(TimeRange(5m, Scan m))` | `value ∈ (0, ∞)`: the `Filter` is above `rate` | | PromQL `quantile(0.99, rate(a[5m]) > rate(b[5m]))` | `BinaryOp(>, rate(TimeRange(5m, Scan a)), rate(TimeRange(5m, Scan b)))` | all of it | nothing: a vector/vector comparison is a `BinaryOp`, not a `Filter` | @@ -405,7 +468,7 @@ Any other shape stays in `definition`: #### 4.2.3 Cost of deriving coverage -**When it runs.** `coverage()` derives a node's coverage the first time it is called and caches it on the node, so each node pays once. Building a `SummaryMerge` (`OperatorNode::new`) derives its coverage to reject an invalid merge and caches the result, so `coverage()` and `validate_structure` do not derive it again. `validate_structure` derives it only for a merge that was not built through `new`. +**When it runs.** `find_or_add_group` derives an aggregate group's coverage once, when it creates the group, and stores it in `GroupProperties::aggregate_coverage`; `add_candidate` derives a candidate's the same way. A `SummaryMerge` reads its inputs' stored coverage, so each state is derived once. Operator-node trees carry no coverage. **Sizes used below.** @@ -688,7 +751,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ State: series labels, state Sketch(KLL k=200) │ [[ SummaryMerge ]] - ▲ selection: time (−2m, 0] + ▲ selection: ts ∈ (t − 2m, t] │ ┌─────────────────┴─────────────────┐ │ State: series labels, │ State: series labels, @@ -696,7 +759,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ │ [[ SummaryAgg ]] tumbling window 0 [[ SummaryAgg ]] tumbling window 1 KLL k=200, per series KLL k=200, per series - selection: time (−1m, 0] selection: time (−2m, −1m] + selection: ts ∈ (t − 1m, t] selection: ts ∈ (t − 2m, t − 1m] ▲ ▲ │ │ [ TimeRange 1m ] [ TimeRange 1m ] @@ -798,26 +861,26 @@ input groups output groups | State | Query | Selection | |---|---|---| -| `A` | `quantile_over_time(0.99, m[1m])` | time `(−1m, 0]` | -| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | time `(−2m, −1m]` | -| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | time `(−90s, −30s]` | -| `D` | `quantile_over_time(0.99, n[1m])` | time `(−1m, 0]`, but over metric `n`: definition `KLL(value) per series over Scan n` | +| `A` | `quantile_over_time(0.99, m[1m])` | ts ∈ `(t − 1m, t]` | +| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | ts ∈ `(t − 2m, t − 1m]` | +| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | ts ∈ `(t − 90s, t − 30s]` | +| `D` | `quantile_over_time(0.99, n[1m])` | ts ∈ `(t − 1m, t]`, but over metric `n`: definition `KLL(value) per series over Scan n` | | Merge | Allowed? | Why | Result's selection | |---|---|---|---| -| `A + B` | ✓ | same definition, no overlap | `(−2m, 0]` (touching time windows join) | -| `A + C` | ✗ | `(−60s, −30s]` is in both, so those rows would be counted twice | | +| `A + B` | ✓ | same definition, no overlap | `(t − 2m, t]` (touching time windows join) | +| `A + C` | ✗ | `(t − 60s, t − 30s]` is in both, so those rows would be counted twice | | | `A + A` | ✗ | every row is in both | | | `A + D` | ✗ | different definitions: `D` reads metric `n`, not `m` | | | `A` + a KLL with `k = 400` | ✗ | different schema: `k` is part of the state type, so the merge is rejected before coverage is checked | | -| `(A + B)` + the state of `m[1m] offset 2m`, time `(−3m, −2m]` | ✓ | a merge has coverage like any state, so merges nest | `(−3m, 0]` | +| `(A + B)` + the state of `m[1m] offset 2m`, ts ∈ `(t − 3m, t − 2m]` | ✓ | a merge has coverage like any state, so merges nest | `(t − 3m, t]` | **Whether inputs may overlap** depends on the summary type (family and algorithm, §3). This is #592 (open), which adds `FieldDataType::family_merges` and `merge_relation`; #646 alone requires every merge to be overlap-free: | Rule | Summary types | Example | |---|---|---| -| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `(−60s, −30s]` would be counted twice | -| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `(−1m, 0]` and `(−90s, −30s]`, which together cover `(−90s, 0]` | +| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `(t − 60s, t − 30s]` would be counted twice | +| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `(t − 1m, t]` and `(t − 90s, t − 30s]`, which together cover `(t − 90s, t]` | | **cannot merge (yet)** | `CmsWithHeap`, `CountSketchWithHeap`, Theta, KMV, exact `Rate`/`IRate`/`Increase`, samples, wavelets, models | no sound merge is modeled yet, so #592 rejects them | | **right inside left** | subtraction | 5.7 | @@ -825,7 +888,7 @@ input groups output groups |---|---| | What comes out? | the children's schema; with `group_by`, its columns in order, then the state column. A `group_by` of no columns over a SQL relation's states stands for a global SQL aggregate, which has one row over no input rows: it outputs one state, empty when there is no input state | | When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra) | -| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(−2m, −1m]` and `(−1m, 0]` give `(−2m, 0]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | +| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(t − 2m, t − 1m]` and `(t − 1m, t]` give `(t − 2m, t]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | | State or value? | state in, state out | **Guarantee after a rollup.** The groups of one build partition its rows, so the states a rollup merges into one output group are disjoint, and the family-capability table's merged error applies. A rolled-up state of these families keeps the guarantee one build of the same size would have over the coarse group's rows; most are exactly that build's state: @@ -847,7 +910,7 @@ Rejected: counter states (`Rate`, `Increase`), which merge consecutive windows o **Other uses of coverage.** The planner also uses coverage to read or reuse a state without merging: - **Slice:** from a `by[region, job]` state, a query for `region = 'us'` by job reads only the `us` groups ✓. A query for `value < 50` ✗: `value` is not a grouping column, and a sketch cannot be filtered after it is built. -- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `(−5m, 0]` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. +- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `(t − 5m, t]` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. ### 5.7 Reserved operators (not implemented) @@ -888,9 +951,10 @@ OperatorNode §6.1 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output │ └── Sketch(SketchAlgorithm) `state` field has type `summary_type`) │ (θ, `SketchParams`, is on the candidate) §6.4 -├── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) -└── coverage() → SummaryCoverage §6.6 - └── definition: OperatorNode (again a node, so the same structure) +└── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) + +MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 + └── definition: AggregateDefinition { input, unlifted_operators } ``` | Subsection | Defines | Used by | @@ -900,15 +964,15 @@ OperatorNode §6.1 | §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.summary_type` | | §6.4 Summary algorithms and parameters | `SketchAlgorithm`, `SketchParams`, `HydraParams`, exact/sample/wavelet/model kinds and params | the payload of each non-plain `FieldDataType` variant; θ on a candidate's `LogicalProperties` | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | -| §6.6 Summary coverage | `SummaryCoverage`, `SelectionBox`, `Constraint` | `OperatorNode::coverage()` on summary nodes | +| §6.6 Aggregate coverage | `AggregateCoverage`, `AggregateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of aggregate memo groups | -**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). +**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `AggregateCoverage` (§6.6). ### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) Every operator in a DAG is wrapped in an `OperatorNode`. `coverage()` is explained in §6.6. -Shown as of #646. On `main` today the node still has a declared `pub coverage: Option` field (with `with_coverage()` and `requires_coverage()`); #646 replaces it with the derived `coverage()` below. +Shown as of #646. On `main` today the node still has a declared `pub coverage: Option` field (with `with_coverage()` and `requires_coverage()`); #646 replaces it with the derived `coverage()` below. ```rust /// One node of the DAG. Immutable and shared through `Rc`. @@ -933,7 +997,7 @@ impl OperatorNode { /// The node's coverage (§4.2), derived on first use and then cached. /// `Some` for a `SummaryAgg` and a valid `SummaryMerge`; `None` for every /// other node. - pub fn coverage(&self) -> Option<&SummaryCoverage>; + pub fn coverage(&self) -> Option<&AggregateCoverage>; } ``` @@ -1311,18 +1375,18 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Summary coverage (`crates/types/src/ir/properties/summary_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/ir/properties/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; the plan's content id in `crates/types/src/stages/logical/aggregate_coverage.rs`) -The code for §4, as of #646. On `main` today `SummaryCoverage` is still the older declared form (`source` plus `regions`). +The code for §4. `AggregateDefinition` is shown in §4.2.2. ```rust -pub struct SummaryCoverage { - /// What the state computes: the `SummaryAgg` and its sub-DAG, with the - /// conditions that went into `selection` taken out (§4.2.2). - pub definition: Rc, +pub struct AggregateCoverage { + /// What the state computes: its input and the operators the walk did + /// not lift (§4.2.2). + pub(crate) definition: AggregateDefinition, /// Which rows went in: a union of boxes. A row is in the state if it is /// in at least one box. - pub selection: Vec, + pub(crate) selection: Vec, } /// One box: a row is in it when it meets every constraint. @@ -1375,10 +1439,11 @@ A PromQL range `m[r]` evaluated at t reads the samples in (t − r, t]: left-ope | SQL `WHERE price > 10` | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | ```rust -impl SummaryCoverage { - /// Computes the coverage of a summary node from its sub-DAG (§4.2.2). - /// Errors: `NotSummary` for a node that is not a `SummaryAgg`, - /// `SummaryTumblingWindowChain` or `SummaryMerge`; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` +impl Operator { + /// The coverage of what this operator computes over its inputs + /// (§4.2.2). Errors: `NoCoverage` for an operator that is + /// not a `SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge` or a + /// `Filter` keeping whole states; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` /// that does not expand to one build over its whole window; for a /// merge, an input's own error, `EmptyMerge` (no inputs), /// `DefinitionMismatch` (inputs compute different things) or @@ -1388,9 +1453,15 @@ impl SummaryCoverage { /// not a group key of the inputs' build), `FamilyCannotRegroup` (counter /// states, a family without a merge, or a Hydra) or `RegroupedBuild` /// (the coarse build it stands for does not derive). - pub fn derive(node: &OperatorNode) -> Result; + pub fn aggregate_coverage(&self) -> Result, CoverageError>; } + +/// The plan-scope coverage of `group`: the content id of its expanded +/// definition and its selection; `None` for a group without aggregate +/// coverage. Each input group is expanded to its expression, recursively +/// down to the `Scan`s (`map_references`), so the id names no group. +pub fn view_coverage(memo: &Memo, group: MemoGroupId) -> Option; ``` -- `OperatorNode::new` and `validate_structure` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap; #592 relaxes the overlap check per family). The derived coverage is cached on the node. +- `find_or_add_group` and `add_candidate` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap, per the family's `MergeInputRule`). Operator-node trees have no coverage and no merge check. - A `SummaryAgg` always has coverage: what cannot go into `selection` stays in `definition`. diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 2468e8daa..d4e04bcbe 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -17,7 +17,7 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | Shared vocabulary, by scope: the operator IR by IR section, `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator arguments, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`), `ir::properties` (execution timing, summary coverage) and `ir::flat`; summary types (`summary_type`: `SketchAlgorithm`, `SketchParams`, `SummaryTypeCapability`); accuracy targets and guarantees (`accuracy`: `AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`); the planner inputs (`workload`); the stage-boundary types (`stages`, with Stage 2's `PhysicalProperties` (timing and `Lifetime`)); and Stage 3's plan (`plan`: `plan::physical_asap_dag`, `plan::materialized_view`) | +| `asap-types` (`crates/types`) | Shared vocabulary, by scope: the operator IR by IR section, `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator arguments, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`), `ir::properties` (execution timing, the selection algebra of aggregate coverage) and `ir::flat`; summary types (`summary_type`: `SketchAlgorithm`, `SketchParams`, `SummaryTypeCapability`); accuracy targets and guarantees (`accuracy`: `AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`); the planner inputs (`workload`); the stage-boundary types (`stages`, with Stage 2's `PhysicalProperties` (timing and `Lifetime`)); and Stage 3's plan (`plan`: `plan::physical_asap_dag`, `plan::materialized_view`) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | | `asap-stage0-memo` (`crates/stage0-memo`) | Stage 0 hash-consing: the memo groups of the workload's query roots (`memo_of_queries`) | | `asap-stage1-pass1-per-computation-candidates` (`crates/stage1-pass1-per-computation-candidates`) | Stage 1, Pass 1: the initial candidates of each memo group | diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 4af0ad108..d57d603a4 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -27,7 +27,7 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- needs two timings; `split_shared_by_timing` copies it for one side first) or `IllegalChildDataState`. These are internal errors in a candidate the stages built, not bad user input. -- A summary build's `SummaryCoverage` (which rows and time range its state +- A summary build's `AggregateCoverage` (which rows and time range its state represents) is derived, never declared; a merge's inputs must be disjoint as its family's `MergeInputRule` requires, or coverage fails (`InputsOverlap`).