diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index 3cc9a088e..14177c829 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -1,5 +1,5 @@ //! How the derivation rules read an operator's inputs: one trait, so the -//! same rules (output schema and kind, input validation, sharing, summary +//! same rules (output schema and kind, input validation, sharing, aggregate //! coverage) derive a front-end node's schema and a memo group's //! `GroupProperties` (concepts doc `memo-groups-properties-requirements.md`, //! section 3). diff --git a/crates/types/src/ir/properties/aggregate_coverage.rs b/crates/types/src/ir/properties/aggregate_coverage.rs index fbfca1122..c8489251d 100644 --- a/crates/types/src/ir/properties/aggregate_coverage.rs +++ b/crates/types/src/ir/properties/aggregate_coverage.rs @@ -144,70 +144,63 @@ pub enum CoverageError { } impl Operator { - /// The coverage of what this operator computes over its inputs - /// (`coverage_of`, below). + /// Coverage of what this operator computes over its inputs: a + /// `SummaryAgg`, an exact `Aggregate`, a + /// `SummaryTumblingWindowChain` (a build over its input, the timestamp + /// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the + /// chain's coverage with the windows' interval on the timestamp column), + /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped + /// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` + /// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge + /// reads its inputs' stored coverage and fails + /// unless every input has the same definition and their selections are + /// pairwise disjoint; a family whose merge is idempotent + /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may + /// also merge overlapping selections, and a counter family + /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. pub fn aggregate_coverage(&self) -> Result, CoverageError> { - coverage_of(self) - } -} - -/// Coverage of what `operator` computes over its inputs: a `SummaryAgg`, an exact `Aggregate`, a -/// `SummaryTumblingWindowChain` (a build over its input, the timestamp -/// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the -/// chain's coverage with the windows' interval on the timestamp column), -/// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped -/// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` -/// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge -/// reads its inputs' stored coverage and fails -/// unless every input has the same definition and their selections are -/// pairwise disjoint; a family whose merge is idempotent -/// ([`MergeInputRule::MayOverlap`] in the family-capability table) may -/// also merge overlapping selections, and a counter family -/// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. -fn coverage_of( - operator: &Operator, -) -> Result, CoverageError> { - let as_built = |op: &ASAPOp| -> ASAPOp> { - op.map_children(|input| InputOrTemporaryBuild::Input(input.clone())) - }; - match operator { - Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { - let top = unlifted( - &operator.map_children(|input| InputOrTemporaryBuild::Input(input.clone())), - ); - of_aggregate(&InputOrTemporaryBuild::Input(child.clone()), None, |_| top) - } - Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - if operator.output_kind() == OperatorResultKind::State => - { - of_window_selection(operator) - } - Operator::NonASAP(_) => match selected_build(operator)? { - Some(build) => of_summary_agg(&build), - None => Err(CoverageError::NoCoverage), - }, - Operator::ASAP( - op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), - ) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryMerge { - children, group_by, .. - }) => { - let inputs = children - .iter() - .map(|child| { - child - .aggregate_coverage() - .cloned() - .ok_or(CoverageError::NoCoverage) - }) - .collect::, _>>()?; - let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; - match group_by { - None => Ok(merged), - Some(keys) => regrouped(merged, &children[0], keys), + let as_built = |op: &ASAPOp| -> ASAPOp> { + op.map_children(|input| InputOrTemporaryBuild::Input(input.clone())) + }; + match self { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = unlifted( + &self.map_children(|input| InputOrTemporaryBuild::Input(input.clone())), + ); + of_aggregate(&InputOrTemporaryBuild::Input(child.clone()), None, |_| top) } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if self.output_kind() == OperatorResultKind::State => + { + of_window_selection(self) + } + Operator::NonASAP(_) => match selected_build(self)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NoCoverage), + }, + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => of_summary_agg(&as_built(op)), + Operator::ASAP(ASAPOp::SummaryMerge { + children, group_by, .. + }) => { + let inputs = children + .iter() + .map(|child| { + child + .aggregate_coverage() + .cloned() + .ok_or(CoverageError::NoCoverage) + }) + .collect::, _>>()?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; + match group_by { + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), + } + } + Operator::ASAP(_) => Err(CoverageError::NoCoverage), } - Operator::ASAP(_) => Err(CoverageError::NoCoverage), } } @@ -611,7 +604,7 @@ fn residual( Some((column, constraint)) => { lifted.insert(column, constraint); } - None => rest.push(conjunct.map_operator_refs(&mut reference_of)), + None => rest.push(conjunct.map_operator_refs(&mut OperatorInput::reference)), } } match rest.len() { @@ -630,17 +623,12 @@ fn combined_with(existing: Option<&Constraint>, constraint: Constraint) -> Optio } } -/// How a definition names `input`; `None` for a rule's temporary build. -fn reference_of(input: &InputOrTemporaryBuild) -> Option { - input.reference() -} - /// `link`, a liftable operator or an exact `Aggregate` kept whole in a /// definition: its main input `None`, every other input its reference. fn unlifted( link: &Operator>, ) -> Operator> { - let mut operator = link.map_children(reference_of); + let mut operator = link.map_children(OperatorInput::reference); match &mut operator { Operator::NonASAP( NonASAPOp::Filter { child, .. } diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index a60c88cfe..23598bd52 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; -use super::MemoGroupAsInput; +use super::group::MemoGroupAsInput; use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction}; use crate::ir::operator::asap::{summary_agg_layout, summary_build}; use crate::ir::operator::input::OperatorInput; diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index 5e436c7a9..e64e092c5 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -9,7 +9,7 @@ use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{ProjectItem, SortKey}; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; use crate::ir::{ExprSemantics, NonASAPOp, Predicate, ScalarExpr}; -use crate::stages::logical::MemoGroupAsInput; +use crate::stages::logical::group::MemoGroupAsInput; use crate::stages::{CandidateExpression, LogicalProperties, OriginOfCandidateForMemoGroup}; use crate::summary_type::{SketchAlgorithm, SketchParams, SketchStatistic}; use crate::test_support::memo_fixtures::{aggregate_where, readout, scan, state}; diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 59ba58c11..386069b95 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -1,7 +1,6 @@ //! The logical side: Stage 0's and Stage 1's memo, by scope (memo → group //! → candidate → candidate input → edge → Stage 1's output). -mod aggregate_coverage; mod candidate; mod column_properties; mod edge; @@ -9,16 +8,16 @@ mod group; mod memo; mod memo_candidates; mod requirements; +mod view_coverage; -pub use aggregate_coverage::view_coverage; pub use candidate::{ CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, }; pub use column_properties::ValueSign; pub use edge::LogicalEdge; -use group::MemoGroupAsInput; pub use group::{GroupProperties, MemoGroup, MemoGroupId}; pub use memo::{Memo, MemoQuery}; pub use memo_candidates::LogicalASAPMemoCandidates; pub use requirements::{LogicalRequirements, RequiredAccuracy}; +pub use view_coverage::view_coverage; diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/view_coverage.rs similarity index 83% rename from crates/types/src/stages/logical/aggregate_coverage.rs rename to crates/types/src/stages/logical/view_coverage.rs index a8a80d550..8d1b3c7ed 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/view_coverage.rs @@ -1,8 +1,9 @@ -//! Memo scope of aggregate coverage: the IR derives it -//! ([`crate::ir::properties::aggregate_coverage`]) over memo groups -//! (`R` = [`MemoGroupId`]) and the memo stores it per group -//! (`GroupProperties::aggregate_coverage`); here the plan's view of it, a -//! content id that names no group ([`view_coverage`]). +//! A memo group's view coverage ([`view_coverage`]): the plan's content id +//! for the group's aggregate coverage, which names no group. The IR derives +//! aggregate coverage ([`crate::ir::properties::aggregate_coverage`]) over +//! memo groups (`R` = [`MemoGroupId`]) and the memo stores it per group +//! (`GroupProperties::aggregate_coverage`); this file expands the stored +//! coverage's input groups and hashes the result. use serde::Serialize; diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index cfa260d4c..6022923ea 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -513,7 +513,7 @@ This section walks through each summary operator with one small example. For eac |---|---| | **What comes out?** | the output schema the planner derives (`output_schema()`) | | **When is it rejected?** | the checks the planner runs when it builds the node | -| **What is its coverage?** | `coverage()` of the output (§4.2) | +| **What is its coverage?** | the aggregate coverage of the output (§4.2), derived by `Operator::aggregate_coverage` | | **State or value?** | whether the output is summary state or a readable value | **How to read the diagrams.** Data flows from bottom to top, along the `▲` arrows. Each edge is labelled with the schema it carries, written `Kind: field Type, …`. @@ -523,7 +523,7 @@ This section walks through each summary operator with one small example. For eac | `[ Op ]` | operator whose output is a value (`Relation`) | | `[[ Op ]]` | operator whose output is summary state (`State`) | | `( next operator )` | whatever consumes the result | -| `selection: …` next to a node | the `selection` part of that node's `coverage()` | +| `selection: …` next to a node | the `selection` part of that node's aggregate coverage | All examples read a table, so values are `Relation`. For PromQL series they would be `InstantVector`. A *closed schema* lists every column of the table. For readability, a filter on a table is drawn as a `Filter` over the `Scan`; the frontend folds it into `Scan.predicates`, which gives the same coverage (§4.2.2). @@ -556,7 +556,7 @@ With an error target, the planner answers the quantile from a KLL sketch with `k │ [ Scan t ] -coverage() of the SummaryAgg +aggregate coverage of the SummaryAgg ┌──────────────────────────────────────────────────────────────────┐ │ definition: this SummaryAgg over Scan t (the Filter removed) │ │ selection: region ∈ {us}, latency ∈ (−∞, 10000) │ @@ -639,7 +639,7 @@ The planner answers `SUM` with an exact `Sum` accumulator: a `SummaryAgg` builds │ [ Scan t ] -coverage() of the SummaryAgg +aggregate coverage of the SummaryAgg ┌──────────────────────────────────────────────────┐ │ definition: this SummaryAgg over Scan t │ │ selection: everything (no filter) │ @@ -966,13 +966,11 @@ MemoGroup.group_properties.aggregate_coverage: AggregateCoverage §6.6 | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | | §6.6 Aggregate coverage | `AggregateCoverage`, `AggregateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of aggregate memo groups | -**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `AggregateCoverage` (§6.6). +**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `Operator::aggregate_coverage` over the `SummaryAgg` returns an `AggregateCoverage` (§6.6), which the memo stores in the group's `GroupProperties::aggregate_coverage`. ### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) -Every operator in a DAG is wrapped in an `OperatorNode`. `coverage()` is explained in §6.6. - -Shown as of #646. On `main` today the node still has a declared `pub coverage: Option` field (with `with_coverage()` and `requires_coverage()`); #646 replaces it with the derived `coverage()` below. +Every operator in a DAG is wrapped in an `OperatorNode`. A node stores no coverage: coverage is derived by the IR rule `Operator::aggregate_coverage` over `OperatorInput` and stored per memo group in `GroupProperties::aggregate_coverage` (§6.6). ```rust /// One node of the DAG. Immutable and shared through `Rc`. @@ -987,17 +985,6 @@ pub struct OperatorNode { /// The accuracy statement of the output, once known. `None` does not /// mean exact. pub guarantee: Option, - /// Cache for `coverage()`. Filled on first use, never serialized, - /// ignored by equality, emptied on clone. Not a source of truth: - /// coverage can always be derived again. - coverage_cache: CoverageCache, -} - -impl OperatorNode { - /// The node's coverage (§4.2), derived on first use and then cached. - /// `Some` for a `SummaryAgg` and a valid `SummaryMerge`; `None` for every - /// other node. - pub fn coverage(&self) -> Option<&AggregateCoverage>; } ``` @@ -1375,7 +1362,7 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Aggregate coverage (`crates/types/src/ir/properties/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; the plan's content id in `crates/types/src/stages/logical/aggregate_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/ir/properties/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. @@ -1450,7 +1437,7 @@ impl Operator { /// `InputsOverlap { required }` (inputs not proven disjoint, or disjoint /// in time for counter states, as the family's `MergeInputRule` /// requires); for a rollup, also `NotAGroupKey` (a `group_by` column is - /// not a group key of the inputs' build), `FamilyCannotRegroup` (counter + /// not a group key of the inputs' build), `SummaryTypeCannotRegroup` (counter /// states, a family without a merge, or a Hydra) or `RegroupedBuild` /// (the coarse build it stands for does not derive). pub fn aggregate_coverage(&self) -> Result, CoverageError>;