diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index 676b5805e..cdd8e3782 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -954,12 +954,7 @@ fn with_root_estimate( estimate: Rc, dropped: &[String], ) -> Result, String> { - let Some(NonASAPOp::Project { - cols, - qualifier, - child, - }) = root.non_asap() - else { + let Some(NonASAPOp::Project { cols, child }) = root.non_asap() else { return Ok(estimate); }; let mut kept = Vec::new(); @@ -980,7 +975,6 @@ fn with_root_estimate( let cols = kept; OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { cols, - qualifier: qualifier.clone(), child: estimate, })) .map_err(|e| format!("projection over the new estimate: {e}")) diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index 4d3862e64..440e4db60 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -42,7 +42,6 @@ fn non_asap_samples() -> Vec> { }, NonASAPOp::Project { cols: vec![], - qualifier: None, child: (), }, NonASAPOp::Aggregate { diff --git a/crates/executor/src/expressions/columnar.rs b/crates/executor/src/expressions/columnar.rs index dbb37b317..d718588d2 100644 --- a/crates/executor/src/expressions/columnar.rs +++ b/crates/executor/src/expressions/columnar.rs @@ -813,7 +813,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/aggregate/temporal.rs b/crates/executor/src/operators/aggregate/temporal.rs index b913fad41..12771677e 100644 --- a/crates/executor/src/operators/aggregate/temporal.rs +++ b/crates/executor/src/operators/aggregate/temporal.rs @@ -341,13 +341,11 @@ mod tests { name: "time".into(), dtype: SummaryFamilyType::Plain(DataType::Timestamp), nullable: false, - table: None, }, SummaryField { name: "value".into(), dtype: SummaryFamilyType::Plain(DataType::Float64), nullable: false, - table: None, }, ], time_index: Some(0), diff --git a/crates/executor/src/operators/common.rs b/crates/executor/src/operators/common.rs index f0182757d..0a9732dc5 100644 --- a/crates/executor/src/operators/common.rs +++ b/crates/executor/src/operators/common.rs @@ -15,7 +15,6 @@ pub(super) fn result_field(name: &str, dtype: DataType, nullable: bool) -> Summa name: name.into(), dtype: SummaryFamilyType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/group_table.rs b/crates/executor/src/operators/group_table.rs index 64a173c97..53e6a702d 100644 --- a/crates/executor/src/operators/group_table.rs +++ b/crates/executor/src/operators/group_table.rs @@ -476,7 +476,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: true, - table: None, }; let schema = Arc::new(Schema { fields: vec![ @@ -548,7 +547,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let schema = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/src/operators/series_labels.rs b/crates/executor/src/operators/series_labels.rs index f44a87933..fdf8b8b73 100644 --- a/crates/executor/src/operators/series_labels.rs +++ b/crates/executor/src/operators/series_labels.rs @@ -1130,7 +1130,6 @@ mod columnar_tests { name: name.into(), dtype: SummaryFamilyType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 66ae98b6b..d7b94f9ac 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -92,7 +92,6 @@ impl Operator { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }); Ok(Self { kind: Kind::AllGroupsSummaryBuild { @@ -176,7 +175,6 @@ impl Operator { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }); Ok(Self { kind: Kind::PerGroupSummaryBuild { diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index 77aa4e5fc..25d221e61 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -617,7 +617,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable, - table: None, } } @@ -663,13 +662,11 @@ mod tests { name: "a".into(), dtype: DataType::Int64, nullable: false, - table: None, }, Field:: { name: "b".into(), dtype: DataType::Utf8, nullable: true, - table: None, }, ]; let labels = DataType::Map { @@ -787,7 +784,6 @@ mod tests { name: "state".into(), dtype: dd_family(), nullable: false, - table: None, }, ], time_index: None, @@ -869,7 +865,6 @@ mod tests { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }, ], time_index: None, @@ -1011,7 +1006,6 @@ mod tests { name: "state".into(), dtype: family, nullable: false, - table: None, }, ], time_index: None, @@ -1064,7 +1058,6 @@ mod tests { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }], time_index: None, unique_keys: vec![], @@ -1143,7 +1136,6 @@ mod tests { name: "state".into(), dtype: family, nullable: false, - table: None, }], time_index: None, unique_keys: vec![], diff --git a/crates/executor/tests/aggregate_memory_limit.rs b/crates/executor/tests/aggregate_memory_limit.rs index 75e6268b9..ac4a75085 100644 --- a/crates/executor/tests/aggregate_memory_limit.rs +++ b/crates/executor/tests/aggregate_memory_limit.rs @@ -53,7 +53,6 @@ fn run_aggregate( name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index 7926c0416..e040021f9 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -20,7 +20,6 @@ fn schema(width: usize) -> SchemaRef { closed: false, fields: (0..width) .map(|i| Field { - table: None, name: format!("v{i}"), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, @@ -193,13 +192,11 @@ fn weighted_summary_build_yields_within_a_batch() { closed: false, fields: vec![ Field { - table: None, name: "item".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, }, Field { - table: None, name: "weight".into(), dtype: FieldDataType::Plain(DataType::Float64), nullable: false, diff --git a/crates/executor/tests/cheap_measures_memory_limit.rs b/crates/executor/tests/cheap_measures_memory_limit.rs index 070950a98..873286fef 100644 --- a/crates/executor/tests/cheap_measures_memory_limit.rs +++ b/crates/executor/tests/cheap_measures_memory_limit.rs @@ -90,7 +90,6 @@ fn many_groups_of_cheap_measures_fail_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![field("k", DataType::Int64), field("v", DataType::Float64)], diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 2b18c3592..87d41f5b6 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -29,7 +29,6 @@ fn schema() -> Arc { ] .into_iter() .map(|(name, dtype)| Field { - table: None, name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index 11fd6e189..f158126be 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -247,7 +247,6 @@ fn heap_sketch_states_load_as_dag_summary_values() { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }], unique_keys: vec![], closed: false, diff --git a/crates/executor/tests/exact_state_columns.rs b/crates/executor/tests/exact_state_columns.rs index 25e89e23d..ce1bcfa61 100644 --- a/crates/executor/tests/exact_state_columns.rs +++ b/crates/executor/tests/exact_state_columns.rs @@ -16,7 +16,6 @@ fn field(name: &str, dtype: DataType) -> Field { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, } } diff --git a/crates/executor/tests/grouped_limit_memory_limit.rs b/crates/executor/tests/grouped_limit_memory_limit.rs index 10f5ceecf..e1b03188a 100644 --- a/crates/executor/tests/grouped_limit_memory_limit.rs +++ b/crates/executor/tests/grouped_limit_memory_limit.rs @@ -43,7 +43,6 @@ fn many_groups_in_one_batch_fail_near_the_memory_limit() { name: "k".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, - table: None, }], time_index: None, unique_keys: vec![], diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index b7ca69528..0696d2816 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -117,7 +117,6 @@ fn summary_agg(update: SummaryUpdate, filter: Option>) -> Physi name: "state".into(), dtype: family(), nullable: false, - table: None, }, ]), ) diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 0733da866..e730108e4 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -157,7 +157,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { name: "state".into(), dtype: heap_family(), nullable: false, - table: None, }, ])) .clone(), diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 517067f4e..b77a7587b 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -26,7 +26,6 @@ fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { fields: fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: FieldDataType::Plain(dtype.clone()), nullable: *nullable, @@ -466,7 +465,6 @@ fn exact_state_and_family_validation() { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "state".into(), dtype: family.clone(), nullable: false, @@ -564,7 +562,6 @@ fn bind_physical_asap_dag_before_execution() { )), }, }], - qualifier: None, child: 0, }), ), diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index a9b0ccc02..1413688fb 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -25,7 +25,6 @@ fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { fields: fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: FieldDataType::Plain(dtype.clone()), nullable: *nullable, @@ -926,7 +925,6 @@ fn planner_node( fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: dtype.clone(), nullable: *nullable, diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index e9af0acad..938cf0b5a 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -23,7 +23,6 @@ fn logical_schema(family: FieldDataType) -> Schema { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "value".into(), dtype: family, nullable: false, diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 29c23131b..d2c03d033 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -25,7 +25,6 @@ fn schema() -> SchemaRef { closed: false, fields: vec![ Field { - table: None, name: "labels".into(), dtype: FieldDataType::Plain(DataType::Map { key: Box::new(DataType::Utf8), @@ -35,7 +34,6 @@ fn schema() -> SchemaRef { nullable: false, }, Field { - table: None, name: "value".into(), dtype: FieldDataType::Plain(DataType::Float64), nullable: false, @@ -346,13 +344,11 @@ fn stored_series_evaluations_support_filters_and_sets() { closed: false, fields: vec![ Field { - table: None, name: PROMQL_SERIES_IDENTITY.into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, }, Field { - table: None, name: "value".into(), dtype: family.clone(), nullable: false, diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 830b799be..10bc11ffe 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -35,7 +35,6 @@ fn fixture() -> (planner_types::ir::NonASAPOp, SchemaRef, Vec) { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "value".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: true, @@ -829,7 +828,6 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { dag.edges.retain(|edge| edge.consumer < 3); let family = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let state = planner_types::ir::schema::Schema::new(vec![Field { - table: None, name: "count".into(), dtype: family.clone(), nullable: false, diff --git a/crates/executor/tests/series_window_memory_limit.rs b/crates/executor/tests/series_window_memory_limit.rs index e8c83bcd5..07ca7d29f 100644 --- a/crates/executor/tests/series_window_memory_limit.rs +++ b/crates/executor/tests/series_window_memory_limit.rs @@ -40,7 +40,6 @@ fn a_long_subquery_over_many_series_fails_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index 6569fcecc..076b8633d 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -27,7 +27,6 @@ fn plain(name: &str, dtype: DataType) -> Field { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, } } diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index f5391cf28..4c385fca6 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -33,13 +33,11 @@ fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { name: "series".into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, - table: None, }, Field { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }, ], time_index: None, diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index 146e067ba..e08176203 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -41,7 +41,6 @@ fn one_large_batch_fails_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 1ab25998e..c624a6d9e 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -85,13 +85,11 @@ fn summary_projection_survives_recovery() { closed: false, fields: vec![ Field { - table: None, name: "state".into(), dtype: family.clone(), nullable: false, }, Field { - table: None, name: "service".into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, @@ -152,7 +150,6 @@ fn summary_projection_survives_recovery() { expr: planner_types::ir::ScalarExpr::Column(index), }) .collect(), - qualifier: None, child: 0, }), output_schema: output.clone(), @@ -251,7 +248,6 @@ fn summary_merge_without_inputs_is_rejected() { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "state".into(), dtype: family, nullable: false, diff --git a/crates/executor/tests/text_extremum_memory.rs b/crates/executor/tests/text_extremum_memory.rs index 7aec609c3..0cc056fa4 100644 --- a/crates/executor/tests/text_extremum_memory.rs +++ b/crates/executor/tests/text_extremum_memory.rs @@ -65,7 +65,6 @@ fn text_max_charges_only_the_current_value() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![field("k", DataType::Int64), field("t", DataType::Utf8)], diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 341977372..49241d6a5 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -158,7 +158,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() false, )])); let state = Arc::new(Schema::new(vec![Field { - table: None, name: "state".into(), dtype: family(), nullable: false, diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 6123568cb..70bcf40f6 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -314,7 +314,6 @@ fn try_rewrite_row_number_top_k( ) -> Result>, SchemaDerivationError> { let Some(NonASAPOp::Project { cols: select_list, - qualifier, child: filter, }) = node.non_asap() else { @@ -427,14 +426,12 @@ fn try_rewrite_row_number_top_k( (Some(partition), Some(order)) if !partition_by.is_without() => { let selected = new(NonASAPOp::Project { cols: kept, - qualifier: qualifier.clone(), child: Rc::clone(ranked), })?; limit(GroupKeys::by(partition), order, selected)? } _ => new(NonASAPOp::Project { cols: kept, - qualifier: qualifier.clone(), child: limit(partition_by.clone(), order_by.clone(), Rc::clone(ranked))?, })?, }; @@ -636,7 +633,6 @@ mod tests { expr: ScalarExpr::Column(1), }, ], - qualifier: None, child, }) } @@ -754,7 +750,6 @@ mod tests { alias: None, expr: ScalarExpr::Column(0), }], - qualifier: None, child: Rc::clone(&ranking), }); let out = canonicalize(Rc::clone(&q)).unwrap(); @@ -982,7 +977,6 @@ mod tests { expr: ScalarExpr::Column(col), }) .collect(), - qualifier: None, child, }) } @@ -1220,7 +1214,6 @@ mod tests { let sub = one_column_subquery(); let root = node(NonASAPOp::Project { child: scan(), - qualifier: None, cols: vec![ ProjectItem { alias: Some("a".into()), diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 79010baa4..1904b798c 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -20,8 +20,8 @@ use std::rc::Rc; use thiserror::Error; -use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction}; -use asap_types::ir::scalar::{resolve_column_ref, resolve_column_refs, ColumnRef, ResolveError}; +use asap_types::ir::operator::{AggIntent, GroupKeys, JoinKind, Reduction}; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ResolveError}; use asap_types::ir::schema::{ColumnId, Schema}; use asap_types::ir::SchemaDerivationError; use asap_types::ir::{NonASAPOp, OperatorNode, Predicate, ProjectItem, ScalarExpr, SortKey}; @@ -43,23 +43,121 @@ pub enum ResolveDAGError { use crate::canonicalize::canonicalize; +/// The names an operator's parent resolves against: the operator's output +/// schema and, per column, the SQL table or alias that qualifies it (`None` +/// when unqualified; [`Scope::unqualified`]'s empty list qualifies no +/// column). A qualified reference (`f.src`) binds here, while lowering; the +/// IR keeps no qualifier. +#[derive(Clone, Copy)] +struct Scope<'a> { + schema: &'a Schema, + qualifiers: &'a [Option], +} + +impl<'a> Scope<'a> { + /// A scope whose columns have no qualifier. + fn unqualified(schema: &'a Schema) -> Self { + Self { + schema, + qualifiers: &[], + } + } + + /// The column `column` names: a qualified name by the column with that + /// qualifier and name, else by the unique column of that name whose + /// qualifier is not known (lost through a `Project` without alias or an + /// aggregate's measures); a column of that name qualified by another + /// table never matches. Any other name by its unique name. + fn resolve(&self, column: &ColumnRef) -> Result { + let ColumnRef::Qualified { table, name } = column else { + return resolve_column_ref(column, self.schema); + }; + let named = |wanted: Option<&str>| { + self.schema + .fields + .iter() + .enumerate() + .filter(|(id, field)| field.name == *name && self.qualifier(*id) == wanted) + .map(|(id, _)| id) + .collect::>() + }; + let qualified = named(Some(table.as_str())); + let candidates = if qualified.is_empty() { + named(None) + } else { + qualified + }; + match candidates.as_slice() { + [id] => Ok(*id), + [] => Err(ResolveError::NotFound { + name: format!("{table}.{name}"), + available: self.schema.fields.iter().map(|f| f.name.clone()).collect(), + }), + _ => Err(ResolveError::Ambiguous { + name: format!("{table}.{name}"), + }), + } + } + + fn resolve_all(&self, columns: &[ColumnRef]) -> Result, ResolveError> { + columns.iter().map(|column| self.resolve(column)).collect() + } + + /// The qualifier of column `column`. + fn qualifier(&self, column: ColumnId) -> Option<&'a str> { + self.qualifiers.get(column).and_then(Option::as_deref) + } +} + +/// A resolved operator with the qualifier of each output column: the scope +/// its parent resolves names in. +struct Bound { + node: Rc, + qualifiers: Vec>, +} + +impl Bound { + /// `node` with `qualifiers`, one per column. + fn new(node: Rc, qualifiers: Vec>) -> Self { + debug_assert_eq!( + qualifiers.len(), + node.schema.fields.len(), + "one qualifier per column" + ); + Self { node, qualifiers } + } + + fn unqualified(node: Rc) -> Self { + let qualifiers = vec![None; node.schema.fields.len()]; + Self::new(node, qualifiers) + } + + fn scope(&self) -> Scope<'_> { + Scope { + schema: &self.node.schema, + qualifiers: &self.qualifiers, + } + } +} + /// Resolve the whole tree rooted at `tree`: bind every `ColumnRef` to a /// `ColumnId` via the usage-derived schema resolver, then canonicalize the result. pub fn resolve_root(tree: &UnresolvedOp) -> Result, ResolveDAGError> { - resolve_root_with_inherited(tree, &[]) + Ok(resolve_root_with_inherited(tree, &[])?.node) } /// [`resolve_root`] with label names inherited from an enclosing scope seeded /// into the leaf schema (a `BinaryOp` side, a scalar operand's operator). +/// Canonicalizing keeps the root's output columns, so their qualifiers. fn resolve_root_with_inherited( tree: &UnresolvedOp, inherited: &[String], -) -> Result, ResolveDAGError> { +) -> Result { let usage_derived_schema = resolve_schema_with_inherited(tree, inherited); let root = resolve(tree, &usage_derived_schema)?; - let root = canonicalize(root)?; - root.validate_structure()?; - Ok(root) + let node = canonicalize(root.node)?; + node.validate_structure()?; + Ok(Bound::new(node, root.qualifiers)) } /// Bind `tree` as a root in its own scope, inheriting from `enclosing` the @@ -73,7 +171,7 @@ fn resolve_nested_root( .into_iter() .filter(|n| !own.contains(n)) .collect(); - resolve_root_with_inherited(tree, &inherited) + Ok(resolve_root_with_inherited(tree, &inherited)?.node) } fn node(op: NonASAPOp) -> Result, ResolveDAGError> { @@ -83,31 +181,33 @@ fn node(op: NonASAPOp) -> Result, ResolveDAGError> { } /// The generic substitution walk. `usage_derived_schema` is the schema a -/// schemaless `Scan` in this scope binds to. -fn resolve( - tree: &UnresolvedOp, - usage_derived_schema: &Schema, -) -> Result, ResolveDAGError> { +/// schemaless `Scan` in this scope binds to. Qualifiers follow the columns: +/// a `Scan` qualifies its columns by its own; an operator that passes +/// columns through, a join and an aggregate's group keys keep them; a +/// `Project` qualifies all its columns by its own or none. +fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result { use UnresolvedOp as U; - let expr = - |e: &UnresolvedScalar, schema: &Schema| resolve_expr_in(e, schema, usage_derived_schema); - let pred = |p: &UnresolvedScalar, schema: &Schema| { - Ok::<_, ResolveDAGError>(Predicate(expr(p, schema)?)) - }; - let sort_keys = |keys: &[UnresolvedSortKey], schema: &Schema| { + let expr = |e: &UnresolvedScalar, scope: Scope| resolve_expr_in(e, scope, usage_derived_schema); + let pred = + |p: &UnresolvedScalar, scope: Scope| Ok::<_, ResolveDAGError>(Predicate(expr(p, scope)?)); + let sort_keys = |keys: &[UnresolvedSortKey], scope: Scope| { keys.iter() .map(|k| { Ok::<_, ResolveDAGError>(SortKey { - expr: expr(&k.expr, schema)?, + expr: expr(&k.expr, scope)?, ascending: k.ascending, nulls_first: k.nulls_first, }) }) .collect::, _>>() }; + // A node over a child with the child's columns and `qualifiers`. + let passing = + |op: NonASAPOp, qualifiers: Vec>| Ok(Bound::new(node(op)?, qualifiers)); + let unqualified = |op: NonASAPOp| Ok(Bound::unqualified(node(op)?)); // A PromQL child over an open schema gets an opaque series-identity column. let promql_series = |child: &UnresolvedOp| -> Result, ResolveDAGError> { - let child = resolve(child, usage_derived_schema)?; + let child = resolve(child, usage_derived_schema)?.node; if child.schema.closed { return Ok(child); } @@ -119,19 +219,28 @@ fn resolve( source, predicates, schema, + qualifier, } => { let schema = schema .clone() .unwrap_or_else(|| usage_derived_schema.clone()); + let qualifiers = vec![qualifier.clone(); schema.fields.len()]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let predicates = predicates .iter() - .map(|p| pred(&p.0, &schema)) + .map(|p| pred(&p.0, scope)) .collect::, _>>()?; - node(NonASAPOp::Scan { - source: source.clone(), - predicates, - schema, - }) + Ok(Bound::new( + node(NonASAPOp::Scan { + source: source.clone(), + predicates, + schema, + })?, + qualifiers, + )) } // Row expressions have no input-column scope. @@ -141,11 +250,11 @@ fn resolve( .iter() .map(|row| { row.iter() - .map(|e| expr(e, &empty)) + .map(|e| expr(e, Scope::unqualified(&empty))) .collect::, _>>() }) .collect::, _>>()?; - node(NonASAPOp::Values { + unqualified(NonASAPOp::Values { rows, schema: schema.clone(), }) @@ -162,7 +271,13 @@ fn resolve( } => { let child = promql_series(child)?; let scalar = resolve_expr(scalar, &Schema::default())?; - lower_scalar_vector(child, scalar, op, *scalar_left, *return_bool) + Ok(Bound::unqualified(lower_scalar_vector( + child, + scalar, + op, + *scalar_left, + *return_bool, + )?)) } U::PromqlMap { child, @@ -171,42 +286,61 @@ fn resolve( } => { let child = promql_series(child)?; let sample = resolve_expr(sample, &child.schema)?; - project_sample(child, sample, *drop_metric_name) + Ok(Bound::unqualified(project_sample( + child, + sample, + *drop_metric_name, + )?)) } - U::PromqlVectorFromScalar(inner) => node(NonASAPOp::PromqlVectorFromScalar(expr( + U::PromqlVectorFromScalar(inner) => unqualified(NonASAPOp::PromqlVectorFromScalar(expr( inner, - usage_derived_schema, + Scope::unqualified(usage_derived_schema), )?)), U::PromqlRelabel { dst, value, child } => { let child = resolve(child, usage_derived_schema)?; - let value = expr(value, &child.schema)?; - node(NonASAPOp::PromqlRelabel { + let value = expr(value, child.scope())?; + unqualified(NonASAPOp::PromqlRelabel { dst: dst.clone(), value, - child, + child: child.node, }) } - U::PromqlInfoEnrich { selector, child } => node(NonASAPOp::PromqlInfoEnrich { - selector: selector.clone(), - child: resolve(child, usage_derived_schema)?, - }), + U::PromqlInfoEnrich { selector, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::PromqlInfoEnrich { + selector: selector.clone(), + child: child.node, + }, + child.qualifiers, + ) + } U::PromqlSeriesSample { by, kind, child } => { let child = resolve(child, usage_derived_schema)?; - let by = resolve_group_keys(by, &child.schema)?; - node(NonASAPOp::PromqlSeriesSample { - by, - kind: *kind, - child, - }) + let by = resolve_group_keys(by, child.scope())?; + passing( + NonASAPOp::PromqlSeriesSample { + by, + kind: *kind, + child: child.node, + }, + child.qualifiers, + ) } U::Filter { pred: p, child } => { let child = resolve(child, usage_derived_schema)?; - let pred = pred(&p.0, &child.schema)?; - node(NonASAPOp::Filter { pred, child }) + let pred = pred(&p.0, child.scope())?; + passing( + NonASAPOp::Filter { + pred, + child: child.node, + }, + child.qualifiers, + ) } U::Project { @@ -220,15 +354,18 @@ fn resolve( .map(|item| { Ok::<_, ResolveDAGError>(ProjectItem { alias: item.alias.clone(), - expr: expr(&item.expr, &child.schema)?, + expr: expr(&item.expr, child.scope())?, }) }) .collect::, _>>()?; - node(NonASAPOp::Project { - cols, - qualifier: qualifier.clone(), - child, - }) + let qualifiers = vec![qualifier.clone(); cols.len()]; + Ok(Bound::new( + node(NonASAPOp::Project { + cols, + child: child.node, + })?, + qualifiers, + )) } U::Aggregate { @@ -240,42 +377,75 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let reduction = resolve_reduction(reduction, &child.schema)?; + let scope = child.scope(); + let reduction = resolve_reduction(reduction, scope)?; let measures = measures .iter() - .map(|m| resolve_agg_intent(m, &child.schema)) + .map(|m| resolve_agg_intent(m, scope)) .collect::, ResolveError>>()?; let filters = filters .iter() - .map(|p| p.as_ref().map(|p| pred(&p.0, &child.schema)).transpose()) + .map(|p| p.as_ref().map(|p| pred(&p.0, scope)).transpose()) .collect::, _>>()?; + // A `by` key keeps its qualifier; a per-series reduction keeps + // its input's columns in place. A top-k emits ranked rows, and a + // measure that adds label columns (`count_values`, `absent`) + // breaks the keys-then-measures layout: no column is qualified. + let out_schema = + aggregate_output_schema(&child.node.schema, &reduction, &measures, output_names)?; + let keyed: Option>> = match &reduction { + _ if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) => None, + Reduction::PerEntity => Some(child.qualifiers.clone()), + Reduction::Reduce(by) if by.is_without() => None, + Reduction::Reduce(by) => Some( + by.keys() + .iter() + .map(|&key| scope.qualifier(key).map(str::to_owned)) + .chain(std::iter::repeat_n(None, measures.len())) + .collect(), + ), + }; + let qualifiers_of = |columns: usize| { + keyed + .clone() + .filter(|keyed| keyed.len() == columns) + .unwrap_or_else(|| vec![None; columns]) + }; // HAVING is evaluated over the aggregate's own output. let having = having .as_ref() .map(|h| { - let out_schema = aggregate_output_schema( - &child.schema, - &reduction, - &measures, - output_names, - )?; - pred(&h.0, &out_schema) + pred( + &h.0, + Scope { + schema: &out_schema, + qualifiers: &qualifiers_of(out_schema.fields.len()), + }, + ) }) .transpose()?; - node(NonASAPOp::Aggregate { + let aggregate = node(NonASAPOp::Aggregate { reduction, measures, output_names: output_names.clone(), filters, having, - child, - }) + child: child.node, + })?; + let qualifiers = qualifiers_of(aggregate.schema.fields.len()); + Ok(Bound::new(aggregate, qualifiers)) } U::Dedup { cols, child } => { let child = resolve(child, usage_derived_schema)?; - let cols = resolve_column_refs(cols, &child.schema)?; - node(NonASAPOp::Dedup { cols, child }) + let cols = child.scope().resolve_all(cols)?; + passing( + NonASAPOp::Dedup { + cols, + child: child.node, + }, + child.qualifiers, + ) } U::Concat { children } => { @@ -283,7 +453,15 @@ fn resolve( .iter() .map(|c| resolve(c, usage_derived_schema)) .collect::, _>>()?; - node(NonASAPOp::Concat { children }) + let qualifiers = children + .first() + .map_or_else(Vec::new, |first| first.qualifiers.clone()); + Ok(Bound::new( + node(NonASAPOp::Concat { + children: children.into_iter().map(|child| child.node).collect(), + })?, + qualifiers, + )) } U::Join { @@ -296,15 +474,32 @@ fn resolve( // sets); the predicate sees left ++ right. let left = resolve_root_with_inherited(left, &[])?; let right = resolve_root_with_inherited(right, &[])?; - let mut concat = left.schema.clone(); - concat.fields.extend(right.schema.fields.iter().cloned()); - let pred = pred(&p.0, &concat)?; - node(NonASAPOp::Join { - kind: kind.clone(), - pred, - left, - right, - }) + let mut concat = left.node.schema.clone(); + concat + .fields + .extend(right.node.schema.fields.iter().cloned()); + let mut qualifiers = left.qualifiers; + qualifiers.extend(right.qualifiers); + let pred = pred( + &p.0, + Scope { + schema: &concat, + qualifiers: &qualifiers, + }, + )?; + // A semi or anti join outputs the left side's columns. + if matches!(kind, JoinKind::Semi | JoinKind::Anti) { + qualifiers.truncate(left.node.schema.fields.len()); + } + Ok(Bound::new( + node(NonASAPOp::Join { + kind: kind.clone(), + pred, + left: left.node, + right: right.node, + })?, + qualifiers, + )) } U::SetOp { @@ -312,12 +507,19 @@ fn resolve( all, left, right, - } => node(NonASAPOp::SetOp { - kind: kind.clone(), - all: *all, - left: resolve_root_with_inherited(left, &[])?, - right: resolve_root_with_inherited(right, &[])?, - }), + } => { + let left = resolve_root_with_inherited(left, &[])?; + let right = resolve_root_with_inherited(right, &[])?; + Ok(Bound::new( + node(NonASAPOp::SetOp { + kind: kind.clone(), + all: *all, + left: left.node, + right: right.node, + })?, + left.qualifiers, + )) + } U::Sort { keys, @@ -325,13 +527,16 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let keys = sort_keys(keys, &child.schema)?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - node(NonASAPOp::Sort { - keys, - partition_by, - child, - }) + let keys = sort_keys(keys, child.scope())?; + let partition_by = resolve_group_keys(partition_by, child.scope())?; + passing( + NonASAPOp::Sort { + keys, + partition_by, + child: child.node, + }, + child.qualifiers, + ) } U::Limit { @@ -341,35 +546,56 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - node(NonASAPOp::Limit { - n: *n, - offset: *offset, - partition_by, - child, - }) + let partition_by = resolve_group_keys(partition_by, child.scope())?; + passing( + NonASAPOp::Limit { + n: *n, + offset: *offset, + partition_by, + child: child.node, + }, + child.qualifiers, + ) } U::PromqlSubquery { range, resolution, child, - } => node(NonASAPOp::PromqlSubquery { - range: *range, - resolution: *resolution, - child: resolve(child, usage_derived_schema)?, - }), - - U::TimeRange { range, kind, child } => node(NonASAPOp::TimeRange { - range: *range, - kind: *kind, - child: resolve(child, usage_derived_schema)?, - }), - - U::TimeShift { shift, child } => node(NonASAPOp::TimeShift { - shift: *shift, - child: resolve(child, usage_derived_schema)?, - }), + } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::PromqlSubquery { + range: *range, + resolution: *resolution, + child: child.node, + }, + child.qualifiers, + ) + } + + U::TimeRange { range, kind, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::TimeRange { + range: *range, + kind: *kind, + child: child.node, + }, + child.qualifiers, + ) + } + + U::TimeShift { shift, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::TimeShift { + shift: *shift, + child: child.node, + }, + child.qualifiers, + ) + } U::SQLWindowFunc { func, @@ -383,19 +609,25 @@ fn resolve( let child = resolve(child, usage_derived_schema)?; let args = args .iter() - .map(|a| expr(a, &child.schema)) + .map(|a| expr(a, child.scope())) .collect::, _>>()?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - let order_by = sort_keys(order_by, &child.schema)?; - node(NonASAPOp::SQLWindowFunc { - func: func.clone(), - args, - partition_by, - order_by, - frame: frame.clone(), - output_name: output_name.clone(), - child, - }) + let partition_by = resolve_group_keys(partition_by, child.scope())?; + let order_by = sort_keys(order_by, child.scope())?; + // The window function's own column comes last, unqualified. + let mut qualifiers = child.qualifiers.clone(); + qualifiers.push(None); + Ok(Bound::new( + node(NonASAPOp::SQLWindowFunc { + func: func.clone(), + args, + partition_by, + order_by, + frame: frame.clone(), + output_name: output_name.clone(), + child: child.node, + })?, + qualifiers, + )) } U::BinaryOp { @@ -414,11 +646,11 @@ fn resolve( .into_iter() .filter(|n| !own.contains(n)) .collect(); - node(NonASAPOp::BinaryOp { + unqualified(NonASAPOp::BinaryOp { operator: operator.clone(), return_bool: *return_bool, - lhs: resolve_root_with_inherited(lhs, &inherited)?, - rhs: resolve_root_with_inherited(rhs, &inherited)?, + lhs: resolve_root_with_inherited(lhs, &inherited)?.node, + rhs: resolve_root_with_inherited(rhs, &inherited)?.node, }) } } @@ -442,28 +674,29 @@ pub(crate) fn resolve_expr( expr: &UnresolvedScalar, schema: &Schema, ) -> Result { - resolve_expr_in(expr, schema, schema) + resolve_expr_in(expr, Scope::unqualified(schema), schema) } -/// [`resolve_expr`] where the operators the expression reads inherit from -/// `enclosing` (the owning root's usage-derived schema) rather than from `schema`. +/// [`resolve_expr`] in `scope`, where the operators the expression reads +/// inherit from `enclosing` (the owning root's usage-derived schema) rather +/// than from the scope's schema. fn resolve_expr_in( expr: &UnresolvedScalar, - schema: &Schema, + scope: Scope, enclosing: &Schema, ) -> Result { use UnresolvedScalar as S; let bx = |e: &UnresolvedScalar| -> Result, ResolveDAGError> { - Ok(Box::new(resolve_expr_in(e, schema, enclosing)?)) + Ok(Box::new(resolve_expr_in(e, scope, enclosing)?)) }; let each = |es: &[UnresolvedScalar]| -> Result, ResolveDAGError> { es.iter() - .map(|e| resolve_expr_in(e, schema, enclosing)) + .map(|e| resolve_expr_in(e, scope, enclosing)) .collect() }; let op = |o: &UnresolvedOp| resolve_nested_root(o, enclosing); Ok(match expr { - S::Column(c) => ScalarExpr::Column(resolve_column_ref(c, schema)?), + S::Column(c) => ScalarExpr::Column(scope.resolve(c)?), S::Literal(s) => ScalarExpr::Literal(s.clone()), S::EvalTimestamp => ScalarExpr::EvalTimestamp, S::CurrentTimestamp => ScalarExpr::CurrentTimestamp, @@ -492,7 +725,7 @@ fn resolve_expr_in( // nanosecond timestamp, which is the same IR type. S::Cast { expr, to, try_cast } => { let operand = bx(expr)?; - if matches!(operand.scalar_type(schema), Ok((dtype, _)) if dtype == *to) { + if matches!(operand.scalar_type(scope.schema), Ok((dtype, _)) if dtype == *to) { *operand } else { ScalarExpr::Cast { @@ -536,8 +769,8 @@ fn resolve_expr_in( .iter() .map(|(w, t)| { Ok(( - resolve_expr_in(w, schema, enclosing)?, - resolve_expr_in(t, schema, enclosing)?, + resolve_expr_in(w, scope, enclosing)?, + resolve_expr_in(t, scope, enclosing)?, )) }) .collect::, ResolveDAGError>>()?, @@ -564,9 +797,9 @@ fn resolve_expr_in( /// Resolve name-based group keys positionally, preserving `by`/`without`. fn resolve_group_keys( keys: &GroupKeys, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { - let ids = resolve_column_refs(keys.keys(), schema)?; + let ids = scope.resolve_all(keys.keys())?; Ok(if keys.is_without() { GroupKeys::without(ids) } else { @@ -587,15 +820,15 @@ fn resolve_group_keys( /// may exist at runtime, and PromQL leaves seed every referenced label via the /// schema resolver, so an unresolved key over an open schema indicates a resolution bug, /// not an absent label. (SQL is unaffected — its `GROUP BY` resolves through -/// the strict [`resolve_column_refs`], and DataFusion has already validated +/// the strict [`resolve_group_keys`], and DataFusion has already validated /// the columns anyway.) fn resolve_group_keys_promql( cols: &[ColumnRef], - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { cols.iter() - .filter_map(|c| match resolve_column_ref(c, schema) { - Err(ResolveError::NotFound { .. }) if schema.closed => None, + .filter_map(|c| match scope.resolve(c) { + Err(ResolveError::NotFound { .. }) if scope.schema.closed => None, other => Some(other), }) .collect() @@ -610,11 +843,11 @@ fn resolve_group_keys_promql( /// path is a no-op difference there. fn resolve_reduction( reduction: &Reduction, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { Ok(match reduction { Reduction::Reduce(by) => { - let ids = resolve_group_keys_promql(by.keys(), schema)?; + let ids = resolve_group_keys_promql(by.keys(), scope)?; Reduction::Reduce(if by.is_without() { GroupKeys::without(ids) } else { @@ -630,18 +863,16 @@ fn resolve_reduction( /// convention); every other field carries through unchanged. fn resolve_agg_intent( intent: &AggIntent, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { let col = |c: &Option| -> Result, ResolveError> { - c.as_ref() - .map(|r| resolve_column_ref(r, schema)) - .transpose() + c.as_ref().map(|r| scope.resolve(r)).transpose() }; Ok(match intent { AggIntent::Count => AggIntent::Count, AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { - left: resolve_column_ref(left, schema)?, - right: resolve_column_ref(right, schema)?, + left: scope.resolve(left)?, + right: scope.resolve(right)?, }, AggIntent::Sum { col: c } => AggIntent::Sum { col: col(c)? }, AggIntent::Min { col: c } => AggIntent::Min { col: col(c)? }, @@ -661,10 +892,7 @@ fn resolve_agg_intent( }, AggIntent::TopK { k } => AggIntent::TopK { k: *k }, AggIntent::Cardinality { cols } => AggIntent::Cardinality { - cols: cols - .iter() - .map(|c| resolve_column_ref(c, schema)) - .collect::>()?, + cols: scope.resolve_all(cols)?, }, AggIntent::FrequencyL2 { col: c } => AggIntent::FrequencyL2 { col: col(c)? }, AggIntent::FrequencyEntropy { col: c } => AggIntent::FrequencyEntropy { col: col(c)? }, @@ -692,7 +920,7 @@ fn resolve_agg_intent( }, AggIntent::HistogramQuantile { q, le } => AggIntent::HistogramQuantile { q: *q, - le: resolve_column_ref(le, schema)?, + le: scope.resolve(le)?, }, AggIntent::Math(f) => AggIntent::Math(f.clone()), AggIntent::Absent { labels } => AggIntent::Absent { @@ -832,11 +1060,7 @@ fn project_sample( } }) .collect(); - node(NonASAPOp::Project { - child, - cols, - qualifier: None, - }) + node(NonASAPOp::Project { child, cols }) } #[cfg(test)] @@ -859,6 +1083,7 @@ mod tests { }, predicates: vec![], schema: None, + qualifier: None, } } @@ -866,6 +1091,47 @@ mod tests { UnresolvedScalar::Column(ColumnRef::Named(n.into())) } + fn qualified(table: &str, name: &str) -> ColumnRef { + ColumnRef::Qualified { + table: table.into(), + name: name.into(), + } + } + + /// A qualified name binds the column of that qualifier; a column of that + /// name qualified by another table never matches, so `x.src` over + /// `[u.src]` is an error, not `u.src`; a column whose qualifier is not + /// known matches by its unique name. + #[test] + fn a_qualified_name_never_binds_another_tables_column() { + let schema = Schema::new(vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("src", DataType::Utf8, false), + Field::plain("dst", DataType::Utf8, false), + ]); + let qualifiers = [Some("f".to_string()), Some("u".to_string()), None]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; + assert_eq!(scope.resolve(&qualified("u", "src")), Ok(1)); + assert!(matches!( + scope.resolve(&qualified("x", "src")), + Err(ResolveError::NotFound { .. }) + )); + assert_eq!(scope.resolve(&qualified("f", "dst")), Ok(2)); + // Before, a unique name matched whatever its qualifier. + let users = Schema::new(vec![Field::plain("src", DataType::Utf8, false)]); + let users_scope = Scope { + schema: &users, + qualifiers: &[Some("u".to_string())], + }; + assert!(matches!( + users_scope.resolve(&qualified("f", "src")), + Err(ResolveError::NotFound { .. }) + )); + } + fn eq_lit(col: UnresolvedScalar, v: &str) -> UnresolvedScalar { UnresolvedScalar::Compare { left: Box::new(col), @@ -888,9 +1154,14 @@ mod tests { #[test] fn resolve_pearson_corr_inputs() { let schema = Schema::new(vec![ - Field::plain("x", DataType::Float64, true).with_table("a"), - Field::plain("x", DataType::Float64, true).with_table("b"), + Field::plain("x", DataType::Float64, true), + Field::plain("x", DataType::Float64, true), ]); + let qualifiers = [Some("a".to_string()), Some("b".to_string())]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let intent = AggIntent::PearsonCorr { left: ColumnRef::Qualified { table: "a".into(), @@ -902,7 +1173,7 @@ mod tests { }, }; assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), + resolve_agg_intent(&intent, scope).unwrap(), AggIntent::PearsonCorr { left: 0, right: 1 } ); let missing = AggIntent::PearsonCorr { @@ -912,7 +1183,7 @@ mod tests { }, right: ColumnRef::Named("missing".into()), }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); + assert!(resolve_agg_intent(&missing, scope).is_err()); } // Every leg resolves independently, qualifiers included; one unknown leg @@ -920,9 +1191,14 @@ mod tests { #[test] fn resolve_distinct_tuple_columns() { let schema = Schema::new(vec![ - Field::plain("k", DataType::Int64, true).with_table("a"), - Field::plain("k", DataType::Int64, true).with_table("b"), + Field::plain("k", DataType::Int64, true), + Field::plain("k", DataType::Int64, true), ]); + let qualifiers = [Some("a".to_string()), Some("b".to_string())]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let qualified = |table: &str| ColumnRef::Qualified { table: table.into(), name: "k".into(), @@ -931,13 +1207,13 @@ mod tests { cols: vec![qualified("b"), qualified("a")], }; assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), + resolve_agg_intent(&intent, scope).unwrap(), AggIntent::Cardinality { cols: vec![1, 0] } ); let missing = AggIntent::Cardinality { cols: vec![qualified("a"), ColumnRef::Named("missing".into())], }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); + assert!(resolve_agg_intent(&missing, scope).is_err()); } // ` > `: the bridged literal comes through unchanged, the @@ -1028,6 +1304,7 @@ mod tests { }, predicates: vec![], schema: Some(input), + qualifier: None, }), }; let resolved = resolve_root(&unresolved).expect("resolves"); @@ -1051,9 +1328,8 @@ mod tests { table_ref: table.into(), }, predicates: vec![], - schema: Some(Schema::new(vec![ - Field::plain("k", DataType::Int64, false).with_table(table) - ])), + schema: Some(Schema::new(vec![Field::plain("k", DataType::Int64, false)])), + qualifier: Some(table.into()), }; let qualified = |table: &str| { UnresolvedScalar::Column(ColumnRef::Qualified { @@ -1093,6 +1369,7 @@ mod tests { source: Source::TimeSeries { metric: "x".into() }, predicates: vec![UnresolvedPredicate(eq_lit(named("a"), "1"))], schema: None, + qualifier: None, }; let unresolved = UnresolvedOp::PromqlScalarOp { child: Rc::new(scan("m")), @@ -1182,7 +1459,7 @@ mod tests { closed: true, }; assert_eq!( - resolve_group_keys_promql(&[ColumnRef::Named("job".into())], &s), + resolve_group_keys_promql(&[ColumnRef::Named("job".into())], Scope::unqualified(&s)), Ok(vec![]) ); // Present keys still resolve positionally; absent ones drop around them. @@ -1192,7 +1469,7 @@ mod tests { ColumnRef::Named("job".into()), ColumnRef::Named("group".into()) ], - &s + Scope::unqualified(&s) ), Ok(vec![0]) ); @@ -1205,7 +1482,7 @@ mod tests { // absent label. Keep the strict error. let open = ts_value_schema(); // closed: false assert!(matches!( - resolve_group_keys_promql(&[ColumnRef::Named("job".into())], &open), + resolve_group_keys_promql(&[ColumnRef::Named("job".into())], Scope::unqualified(&open)), Err(ResolveError::NotFound { .. }) )); } diff --git a/crates/frontend-common/src/schema_resolver.rs b/crates/frontend-common/src/schema_resolver.rs index 5462304cc..96be18f6e 100644 --- a/crates/frontend-common/src/schema_resolver.rs +++ b/crates/frontend-common/src/schema_resolver.rs @@ -222,6 +222,7 @@ mod tests { }, predicates: vec![], schema: None, + qualifier: None, } } diff --git a/crates/frontend-common/src/unresolved.rs b/crates/frontend-common/src/unresolved.rs index 52255beeb..0e55023d4 100644 --- a/crates/frontend-common/src/unresolved.rs +++ b/crates/frontend-common/src/unresolved.rs @@ -114,6 +114,10 @@ pub enum UnresolvedOp { /// `Some` for a catalog-backed (SQL) leaf; `None` defers to the /// usage-derived schema resolver. schema: Option, + /// The SQL table or alias qualifying the leaf's columns, which + /// qualified references (`t.col`) above resolve by; `None` for a + /// PromQL leaf. + qualifier: Option, }, Values { rows: Vec>, @@ -125,6 +129,9 @@ pub enum UnresolvedOp { }, Project { cols: Vec, + /// The alias of a derived table (`FROM (SELECT …) t`) qualifying + /// every output column, which qualified references above resolve + /// by; `None` for an ordinary SELECT list. qualifier: Option, child: Rc, }, diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs index 23df96169..4c38d3ce7 100644 --- a/crates/frontend-metricsql/src/lib.rs +++ b/crates/frontend-metricsql/src/lib.rs @@ -153,6 +153,7 @@ impl Lowerer<'_> { .map(|f| UnresolvedPredicate(matcher(f))) .collect(), schema: None, + qualifier: None, }) } diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 942593d70..731d3b0c3 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -1671,6 +1671,7 @@ fn filtered_source(metric: String, matchers: Vec, shift: TimeShift) -> U predicates: matchers.into_iter().map(UnresolvedPredicate).collect(), // Usage-derived (PromQL is schemaless) — the schema resolver fills this in. schema: None, + qualifier: None, }; if shift.is_identity() { scan diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs index 755e64514..3c4a044a9 100644 --- a/crates/frontend-sql/src/frequency.rs +++ b/crates/frontend-sql/src/frequency.rs @@ -336,7 +336,7 @@ fn sql_frequency_result( name: &str, value: ScalarExpr, ) -> Option> { - let NonASAPOp::Project { qualifier, .. } = root.non_asap()? else { + let NonASAPOp::Project { .. } = root.non_asap()? else { return None; }; let aggregate = |measure, name: &str| { @@ -380,7 +380,6 @@ fn sql_frequency_result( else_expr: Some(Box::new(value)), }, }], - qualifier: qualifier.clone(), child, })) .ok()?; diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs index b015c0032..09edf5759 100644 --- a/crates/frontend-sql/src/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -528,7 +528,7 @@ impl<'a> SqlLowerer<'a> { self.scan_source(&table, &table) } - /// A `Scan` over catalog table `table`, with its columns qualified by + /// A `Scan` over catalog table `table`, its columns qualified by /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so /// `Qualified` column refs resolve to the right side across a join. fn scan_source(&self, table: &str, qualifier: &str) -> Result { @@ -537,24 +537,17 @@ impl<'a> SqlLowerer<'a> { .tables .get(table) .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; - let qualified = Schema { - fields: schema - .fields - .iter() - .cloned() - .map(|c| c.with_table(qualifier)) - .collect(), - time_index: schema.time_index, - unique_keys: schema.unique_keys.clone(), - // Catalog-backed: the table's columns are fully declared → closed. - closed: true, - }; Ok(Unresolved::Scan { source: Source::Table { table_ref: table.to_string(), }, predicates: vec![], - schema: Some(qualified), + schema: Some(Schema { + // Catalog-backed: the table's columns are fully declared → closed. + closed: true, + ..schema.clone() + }), + qualifier: Some(qualifier.to_string()), }) } @@ -1929,12 +1922,14 @@ fn filter_or_fold(pred: Scalar, child: Unresolved) -> Unresolved { source, mut predicates, schema, + qualifier, } => { predicates.push(Predicate(pred)); Unresolved::Scan { source, predicates, schema, + qualifier, } } other => Unresolved::Filter { diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 4ae70b9f9..6650aec43 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -2865,6 +2865,34 @@ async fn lower_err(sql: &str) -> String { .to_string() } +/// Both join sides have a `service` column; the front end binds `h.service` +/// with its own scope while lowering, so the group key is the right side's +/// column, the IR schema carrying no qualifier. +#[tokio::test] +async fn a_qualified_group_key_binds_its_join_side() { + let qe = lower( + "SELECT h.service, SUM(m.bytes) FROM metrics m JOIN hosts h ON m.service = h.service \ + GROUP BY h.service", + ) + .await; + let aggregate = find_aggregate_node(&qe).expect("aggregate"); + let NonASAPOp::Aggregate { + reduction, child, .. + } = op(aggregate) + else { + unreachable!() + }; + let [key] = reduction.expect_reduce().keys() else { + panic!("one key: {reduction:?}") + }; + let metrics_columns = 4; + assert!( + *key >= metrics_columns, + "{key} is a metrics column: {child:#?}" + ); + assert_eq!(child.schema.fields[*key].name, "service"); +} + /// After a join the `Project` drops table qualifiers, so `h.service` must not /// fall back to the first bare `service` (`m.service`): the reference is /// ambiguous and rejected rather than sorting by the wrong column. @@ -2875,7 +2903,7 @@ async fn qualified_order_by_after_a_join_does_not_bind_the_wrong_column() { ORDER BY h.service", ) .await; - assert!(err.contains("several unqualified columns"), "{err}"); + assert!(err.contains("several columns"), "{err}"); } /// `WITHIN GROUP (ORDER BY x DESC)` reads the percentile from the top, as diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index b58aa2ec4..939e8ff63 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -58,7 +58,7 @@ pub use asap_stage3_plan_selection::{ }; pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; pub use asap_types::ir::properties::selection::{ - ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, + Constraint, IntervalConstraintLimit, SelectionBox, }; pub use asap_types::ir::properties::ExecutionTiming; pub use asap_types::plan::materialized_view::{ diff --git a/crates/types/src/accuracy/target.rs b/crates/types/src/accuracy/target.rs index e460ae9af..1b3562729 100644 --- a/crates/types/src/accuracy/target.rs +++ b/crates/types/src/accuracy/target.rs @@ -704,7 +704,6 @@ mod tests { else_expr: Some(Box::new(otherwise)), }, }], - qualifier: None, child: joined.clone(), })) }; diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index e3f090294..35d1be914 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -129,7 +129,6 @@ mod tests { alias: Some("s".into()), expr: ScalarExpr::ScalarSubquery(Rc::clone(&v)), }], - qualifier: None, child: Rc::clone(&v), })) .unwrap(); diff --git a/crates/types/src/ir/operator/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs index 799ce1ade..2449391bf 100644 --- a/crates/types/src/ir/operator/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -499,9 +499,11 @@ impl AggIntent { ), // Ranked rows retain the ranked value's type; the enclosing // operator also preserves item identity via schema::topk. - AggIntent::TopK { k, .. } => { - col(&format!("topk_{k}"), input.dtype.clone(), input.nullable) - } + AggIntent::TopK { k, .. } => col( + &crate::ir::schema::topk::topk_output_name(*k, &[]), + input.dtype.clone(), + input.nullable, + ), AggIntent::Cardinality { .. } => col("cardinality", DataType::Int64, false), AggIntent::FrequencyL2 { .. } => col("frequency_l2", DataType::Float64, false), AggIntent::FrequencyEntropy { .. } => { diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index 2b6ff8f78..ed557c4b6 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -186,6 +186,19 @@ impl Source { } } +/// A column of one scan of a source (a `Scan`'s metric or table), by name: +/// the lineage of a column copied unchanged from a scan, and the key a +/// selection names a column by. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceColumn { + pub source: Source, + /// Which scan of `source`: 0 for the first, counted left to right + /// across joins (a self-join's right side is 1). + pub scan_ordinal: u32, + pub column: String, +} + /// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's /// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison /// operator has exactly one representation (and one `Display`) across the IR. @@ -521,7 +534,7 @@ mod tests { use super::*; use crate::ir::schema::{DataType, Field, Schema}; - /// A key moves to the input's column of the same name and qualifier; a + /// A key moves to the input's column of the same name; a /// key that is missing or ambiguous there, or a `without` grouping, has no /// partitions over the input. #[test] @@ -542,11 +555,6 @@ mod tests { Reduction::by(vec![0]).partitions_over(&inner, &ambiguous), None ); - let qualified = Schema::new(vec![field("job").with_table("t")]); - assert_eq!( - Reduction::by(vec![0]).partitions_over(&inner, &qualified), - None - ); assert_eq!( Reduction::Reduce(GroupKeys::without(vec![0])).partitions_over(&inner, &input), None diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 72bc8ad8f..4f4a0a685 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -136,7 +136,7 @@ impl OperatorNode { Ok(Self::with_schema(operator, schema)) } - /// Build a node with caller-supplied output names and qualifiers. For + /// Build a node with caller-supplied output names. For /// either operator category, `validate_structure` requires all other /// schema metadata to agree with derivation; output kind is derived here. pub fn with_schema(operator: Operator, schema: Schema) -> Self { @@ -205,10 +205,9 @@ impl OperatorNode { /// Replace the children and rebuild: each child is replaced by `f(child)` /// and the node is rebuilt over the new children, re-deriving all - /// structural schema metadata. Only names and qualifiers that override - /// the old derived schema are retained, for either operator category. A - /// change in output arity with such overrides needs an explicit new - /// naming assignment. + /// structural schema metadata. Only names that override the old derived + /// schema are retained, for either operator category. A change in output + /// arity with such overrides needs an explicit new naming assignment. /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, @@ -218,22 +217,15 @@ impl OperatorNode { let mut rebuilt = Self::new(self.operator.map_children(f))?; for (i, (derived, retained)) in previous.fields.iter().zip(&self.schema.fields).enumerate() { - let renamed = retained.name != derived.name; - let requalified = retained.table != derived.table; - if !renamed && !requalified { + if retained.name == derived.name { continue; } if rebuilt.schema.fields.len() != previous.fields.len() { return Err(SchemaDerivationError::InvalidScalarSignature( - "output arity changed; reassign explicit output names and qualifiers".into(), + "output arity changed; reassign explicit output names".into(), )); } - if renamed { - rebuilt.schema.fields[i].name = retained.name.clone(); - } - if requalified { - rebuilt.schema.fields[i].table = retained.table.clone(); - } + rebuilt.schema.fields[i].name = retained.name.clone(); } Ok(rebuilt) } @@ -277,7 +269,7 @@ impl OperatorNode { /// input contract, scalar typing against the owning operator's input /// schema, and agreement between each retained schema and the one /// derived from the operator. Both operator categories may override field - /// names and qualifiers; all structural metadata must match derivation. + /// names; all structural metadata must match derivation. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { for node in Self::reachable(self) { if node.schema.time_index.is_some_and(|i| { @@ -306,7 +298,6 @@ impl OperatorNode { // so new structural metadata cannot accidentally escape validation. for (field, retained) in derived.fields.iter_mut().zip(&node.schema.fields) { field.name = retained.name.clone(); - field.table = retained.table.clone(); } let agree = derived == node.schema; if !agree { diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 2312fd73c..892ce65fb 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -75,14 +75,7 @@ pub enum NonASAPOp> { /// σ — row-level filter. Output schema = child schema. Filter { pred: Predicate, child: C }, /// π — projection. - Project { - cols: Vec>, - /// Re-qualifies every output column with this table alias (a derived - /// table / inline view). `None` for an ordinary SELECT list. - #[serde(default)] - qualifier: Option, - child: C, - }, + Project { cols: Vec>, child: C }, /// γ + α — grouping + aggregate intents. Aggregate { reduction: Reduction, @@ -300,11 +293,7 @@ impl NonASAPOp { child: f(child), } } - Project { - cols, - qualifier, - child, - } => { + Project { cols, child } => { let cols = cols .iter() .map(|c| ProjectItem { @@ -314,7 +303,6 @@ impl NonASAPOp { .collect(); Project { cols, - qualifier: qualifier.clone(), child: f(child), } } @@ -597,16 +585,11 @@ impl NonASAPOp { .. } => { if let [AggIntent::TopK { k, .. }] = measures.as_slice() { - let name = output_names - .first() - .filter(|name| !name.is_empty()) - .cloned() - .unwrap_or_else(|| format!("topk_{k}")); + let name = crate::ir::schema::topk::topk_output_name(*k, output_names); let cols = crate::ir::schema::topk::ranked_row_projection(child, reduction, &name)?; return NonASAPOp::Project { cols, - qualifier: None, child: child.clone(), } .output_schema(); @@ -695,11 +678,7 @@ impl NonASAPOp { // keeps its field verbatim, so an `ExactAggregate` state column // can pass through a projection unchanged; any other expression // is typed against the input and must read plain values. - Project { - cols, - qualifier, - child, - } => { + Project { cols, child } => { let in_schema = child.schema(); let fields: Vec = cols .iter() @@ -707,9 +686,7 @@ impl NonASAPOp { .map(|(i, item)| { let mut field = match &item.expr { ScalarExpr::Column(id) if in_schema.fields.get(*id).is_some() => { - let mut f = in_schema.fields[*id].clone(); - f.table = None; - f + in_schema.fields[*id].clone() } expr => { let (dtype, nullable) = expr.scalar_type(in_schema)?; @@ -720,10 +697,7 @@ impl NonASAPOp { .alias .clone() .unwrap_or_else(|| default_proj_name(&item.expr, i, in_schema)); - Ok(match qualifier { - Some(q) => field.with_table(q), - None => field, - }) + Ok(field) }) .collect::, SchemaDerivationError>>()?; let time_index = fields.iter().position(|c| c.name == "ts"); @@ -1428,7 +1402,6 @@ mod tests { fn project(cols: Vec, child: NonASAPOp) -> NonASAPOp { NonASAPOp::Project { cols, - qualifier: None, child: node(child), } } diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs index 2af73e527..06b7cc02c 100644 --- a/crates/types/src/ir/properties/selection.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -1,7 +1,8 @@ //! The selection algebra of aggregate coverage (a memo group's coverage //! `selection`, in `crate::stages`): which output rows of a computation a //! state took, as a union of boxes ([`SelectionBox`]), each a -//! [`Constraint`] per column ([`ColumnIdentity`]): value sets and intervals, +//! [`Constraint`] per column, named by its source column ([`SourceColumn`]): +//! value sets and intervals, //! with interval limits relative to the evaluation time //! ([`IntervalConstraintLimit`]). A predicate conjunct becomes a constraint //! through `constraint_of`. @@ -11,6 +12,7 @@ use std::ops::Bound; use serde::{Deserialize, Serialize}; +use crate::ir::operator::arguments::SourceColumn; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::ScalarExpr; use crate::ir::schema::ColumnId; @@ -24,24 +26,24 @@ use crate::ir::schema::ColumnId; #[serde(deny_unknown_fields)] pub struct SelectionBox { #[serde(with = "columns_as_list")] - pub(crate) columns: BTreeMap, + pub(crate) columns: BTreeMap, } /// `SelectionBox::columns` as a list of `{column, constraint}`. mod columns_as_list { - use super::{ColumnIdentity, Constraint}; + use super::{Constraint, SourceColumn}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Entry { - column: ColumnIdentity, + column: SourceColumn, constraint: Constraint, } pub(super) fn serialize( - columns: &BTreeMap, + columns: &BTreeMap, serializer: S, ) -> Result { let entries: Vec = columns @@ -56,7 +58,7 @@ mod columns_as_list { pub(super) fn deserialize<'de, D: Deserializer<'de>>( deserializer: D, - ) -> Result, D::Error> { + ) -> Result, D::Error> { let entries = Vec::::deserialize(deserializer)?; Ok(entries .into_iter() @@ -94,26 +96,6 @@ impl From> for Bound { } } -/// A column of the definition child's output, by `(table, name)`. A -/// `Project` below renames its columns and sets their table to its -/// qualifier (none by default). -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ColumnIdentity { - pub(crate) table: Option, - pub(crate) name: String, -} - -#[cfg(any(test, feature = "test-support"))] -impl ColumnIdentity { - pub fn new(table: Option, name: impl Into) -> Self { - Self { - table, - name: name.into(), - } - } -} - /// A constraint on one column of a selection. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -463,12 +445,12 @@ impl Constraint { #[cfg(any(test, feature = "test-support"))] impl SelectionBox { - pub fn new(columns: BTreeMap) -> Self { + pub fn new(columns: BTreeMap) -> Self { Self { columns } } /// The per-column constraints. - pub fn columns(&self) -> &BTreeMap { + pub fn columns(&self) -> &BTreeMap { &self.columns } } @@ -566,6 +548,7 @@ fn touching( #[cfg(test)] mod tests { use super::*; + use crate::ir::operator::arguments::Source; use crate::ir::scalar::ExprSemantics; use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; @@ -590,7 +573,11 @@ mod tests { }; assert!(!last_5m.disjoint(&at)); assert_eq!(last_5m.intersect(&at), None); - let column = ColumnIdentity::new(None, "ts"); + let column = SourceColumn { + source: Source::TimeSeries { metric: "m".into() }, + scan_ordinal: 0, + column: "ts".into(), + }; let one = |constraint: Constraint| SelectionBox { columns: BTreeMap::from([(column.clone(), constraint)]), }; diff --git a/crates/types/src/ir/scalar/column_resolution.rs b/crates/types/src/ir/scalar/column_resolution.rs index 8822c6e85..32425980d 100644 --- a/crates/types/src/ir/scalar/column_resolution.rs +++ b/crates/types/src/ir/scalar/column_resolution.rs @@ -2,8 +2,9 @@ //! //! Front ends emit `ColumnRef` (name-based, optionally table-qualified); the //! IR uses positional [`ColumnId`] resolved against a per-node [`Schema`]. -//! These helpers turn name-based refs into positional ids, qualifier-aware. -//! Front-end name resolution (`asap_frontend_common::resolve`) calls them. +//! These helpers turn name-based refs into positional ids. A schema carries +//! no qualifier, so front-end name resolution (`asap_frontend_common::resolve`) +//! binds a qualified name with its own scope before it calls them. use thiserror::Error; @@ -22,38 +23,26 @@ pub enum ResolveError { NoSampleValue { available: Vec }, #[error("ColumnRef::Wildcard cannot be resolved to a single ColumnId")] WildcardNotPositional, - #[error("column `{name}` matches several unqualified columns")] + #[error("column `{name}` matches several columns")] Ambiguous { name: String }, } /// Resolve a single [`ColumnRef`] to a positional [`ColumnId`]. pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result { match col { - // A bare name must be unique: after a join `k` could be `a.k` or `b.k`. - ColumnRef::Named(name) => match schema.fields.iter().filter(|c| &c.name == name).count() { - 0 => Err(ResolveError::NotFound { - name: name.clone(), - available: schema.fields.iter().map(|c| c.name.clone()).collect(), - }), - 1 => Ok(schema.column_id(name).expect("one field has this name")), - _ => Err(ResolveError::Ambiguous { name: name.clone() }), - }, - // Prefer the (table, name) match; fall back to the bare name for - // schemas whose columns carry no qualifier, but only when that name - // is unique: after a join `b.k` must not bind to `a.k`. - ColumnRef::Qualified { table, name } => { - if let Some(id) = schema.column_id_qualified(table, name) { - return Ok(id); - } + // A name must be unique: after a join `k` could be either side's. + // A schema carries no qualifier, so a qualified name that reaches + // here (an extension intent's payload column, a SQL temporal + // aggregate's argument) resolves by its unique name; the front end + // binds every other qualified name with its own scope. + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => { match schema.fields.iter().filter(|c| &c.name == name).count() { 0 => Err(ResolveError::NotFound { - name: format!("{table}.{name}"), + name: name.clone(), available: schema.fields.iter().map(|c| c.name.clone()).collect(), }), 1 => Ok(schema.column_id(name).expect("one field has this name")), - _ => Err(ResolveError::Ambiguous { - name: format!("{table}.{name}"), - }), + _ => Err(ResolveError::Ambiguous { name: name.clone() }), } } ColumnRef::SampleValue => schema @@ -85,14 +74,6 @@ pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result Result, ResolveError> { - cols.iter().map(|c| resolve_column_ref(c, schema)).collect() -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/ir/scalar/expr_ir.rs b/crates/types/src/ir/scalar/expr_ir.rs index 3ee1dd56d..f66493338 100644 --- a/crates/types/src/ir/scalar/expr_ir.rs +++ b/crates/types/src/ir/scalar/expr_ir.rs @@ -18,9 +18,10 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ColumnRef { Named(String), - /// Table-qualified reference (`t.col` / `alias.col`). Resolved by - /// `(table, name)` so a column name shared across a join (`a.k` vs `b.k`) - /// binds to the correct side. + /// Table-qualified reference (`t.col` / `alias.col`), as a front end + /// emits it. The front end binds it by `(table, name)` with its own + /// scope while it lowers, so a column name shared across a join (`a.k` + /// vs `b.k`) binds to the correct side; a schema carries no qualifier. Qualified { table: String, name: String, diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index d06c82d00..234717316 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -12,7 +12,7 @@ pub mod column_resolution; pub mod expr_ir; pub mod scalar_type_rules; -pub use column_resolution::{resolve_column_ref, resolve_column_refs, ResolveError}; +pub use column_resolution::{resolve_column_ref, ResolveError}; pub use expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use std::rc::Rc; @@ -905,7 +905,6 @@ mod tests { alias: Some("result".into()), expr, }], - qualifier: None, child: OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { table_ref: "t".into(), diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 72e3e5627..a6f65a4cd 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -36,8 +36,10 @@ use crate::summary_type::{ /// joins, and does not own data or prescribe a row/column-oriented storage layout. pub type ColumnId = usize; -/// One field of a [`Schema`]: `name + dtype + nullable`, plus an optional -/// table qualifier. The struct describes a column and holds none of its data. +/// One field of a [`Schema`]: `name + dtype + nullable`. The struct describes +/// a column and holds none of its data. Where a column's values come from is +/// its memo group's column lineage, not part of the field; a front end binds +/// qualified names (`t.col`) with its own scope while it lowers. /// /// `T` is the type vocabulary: [`FieldDataType`] on an operator edge (the /// default, and what [`Schema::fields`] holds), plain [`DataType`] for the @@ -53,46 +55,25 @@ pub struct Field { /// Whether NULL values are allowed in this field. PromQL value /// columns are non-nullable; SQL columns inherit their DDL nullability. pub nullable: bool, - /// Optional table/alias qualifier (SQL `t.col` / `t AS a` → `a`). Travels - /// with the field through joins so a `ColumnRef::Qualified` can pick the - /// right side when both carry the same `name`. `None` for PromQL labels and - /// unqualified columns. - #[serde(default)] - pub table: Option, } impl Field { - /// An unqualified field (`table = None`). pub fn new(name: impl Into, dtype: T, nullable: bool) -> Self { Self { name: name.into(), dtype, nullable, - table: None, } } - /// This field re-qualified under `table` (e.g. by a `SubqueryAlias`). - pub fn with_table(mut self, table: impl Into) -> Self { - self.table = Some(table.into()); - self - } - - /// A reference to this field: qualified by its table when it has one. + /// A reference to this field by its name. pub fn column_ref(&self) -> crate::ir::scalar::ColumnRef { - use crate::ir::scalar::ColumnRef; - match &self.table { - Some(table) => ColumnRef::Qualified { - table: table.clone(), - name: self.name.clone(), - }, - None => ColumnRef::Named(self.name.clone()), - } + crate::ir::scalar::ColumnRef::Named(self.name.clone()) } } impl Field { - /// An unqualified field carrying an ordinary readable value. + /// A field carrying an ordinary readable value. pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self { Self::new(name, FieldDataType::Plain(dtype), nullable) } @@ -129,7 +110,6 @@ impl From> for Field { name: field.name, dtype: FieldDataType::Plain(field.dtype), nullable: field.nullable, - table: field.table, } } } @@ -240,7 +220,7 @@ pub enum DataType { /// catalog tables. Date, /// Variable-length sequence. The existing column contract preserves the - /// element field name, type, and nullability. Nested fields are unqualified. + /// element field name, type, and nullability. List { element: Box> }, /// Ordered named fields, including each field's independent nullability. Struct { fields: Vec> }, @@ -305,7 +285,6 @@ impl Schema { field.name == PROMQL_SERIES_IDENTITY && field.dtype == DataType::Utf8 && !field.nullable - && field.table.is_none() }) } @@ -358,15 +337,6 @@ impl Schema { self.fields.iter().position(|c| c.name == name) } - /// Look up a field by `(table, name)` qualifier — disambiguates columns - /// that share a `name` across a join (`a.k` vs `b.k`). `None` if no field - /// has both that qualifier and name. - pub(crate) fn column_id_qualified(&self, table: &str, name: &str) -> Option { - self.fields - .iter() - .position(|c| c.name == name && c.table.as_deref() == Some(table)) - } - /// Whether this schema has *any* provable unique key. #[cfg(any(test, feature = "test-support"))] pub fn has_unique_key(&self) -> bool { @@ -447,12 +417,4 @@ mod tests { fn schema_without_fields_is_rejected() { assert!(serde_json::from_str::(r#"{"closed":true}"#).is_err()); } - - #[test] - fn qualified_field_serde_roundtrip() { - let c = col("service", DataType::Utf8).with_table("hosts"); - let back: Field = serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap(); - assert_eq!(back, c); - assert_eq!(back.table.as_deref(), Some("hosts")); - } } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 3ad853e70..685e1aa5b 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -56,6 +56,16 @@ pub fn ranked_column(child: &I) -> Result String { + output_names + .first() + .filter(|name| !name.is_empty()) + .cloned() + .unwrap_or_else(|| format!("topk_{k}")) +} + /// Ordered projection of ranked rows: partition keys, ranked value, then /// item identity. TopK retains rows; partition keys alone are not unique. pub(crate) fn ranked_row_projection( diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index 0cbd925a9..a14e9552b 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -13,8 +13,10 @@ //! predicate conjunct moves into `selection` when it can be lifted to the //! top (through `Filter`, a range `TimeRange`, a `TimeShift` and //! direct-column `Project` items) and it is a value set or an interval on -//! one column. The `TimeRange` and `TimeShift`s become one interval on the -//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! one column whose lineage is a source column: a selection names the +//! column by that [`SourceColumn`] (the input groups' +//! `GroupProperties::column_lineage`). The `TimeRange` and `TimeShift`s +//! become one interval on the timestamp column ([`IntervalConstraintLimit`]). Everything else stays in //! `definition` as a residual, so two states are merged only when they //! compute the same thing over disjoint rows. @@ -23,18 +25,19 @@ use std::ops::Bound; use thiserror::Error; +use super::column_properties::{project_lineage, ColumnLineage}; use super::group::MemoGroupAsInput; -use crate::ir::operator::arguments::{AtModifier, Reduction}; +use crate::ir::operator::arguments::{AtModifier, Reduction, SourceColumn}; use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; use crate::ir::properties::selection::{ - constraint_of, union, ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, + constraint_of, union, Constraint, IntervalConstraintLimit, SelectionBox, }; use crate::ir::scalar::expr_ir::ScalarValue; use crate::ir::scalar::{Predicate, ScalarExpr}; -use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::schema::ColumnId; use crate::ir::SchemaDerivationError; use crate::stages::MemoGroupId; use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; @@ -271,7 +274,7 @@ fn selected_build<'m>( /// rows with timestamp column `time_column`. fn of_merge( inputs: &[AggregateCoverage], - time_column: Option<&ColumnIdentity>, + time_column: Option<&SourceColumn>, ) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; if inputs @@ -344,9 +347,12 @@ fn regrouped( if !SummaryTypeCapability::merges_groups(family) { return Err(CoverageError::SummaryTypeCannotRegroup); } - let state = first.schema(); - let rows = build_input_schema(first).ok_or(CoverageError::NoCoverage)?; - // The build's input column each key of the state names. + let state = first.group_properties(); + let rows = build_input(first).ok_or(CoverageError::NoCoverage)?; + let rows_lineage = &rows.group_properties().column_lineage; + // The build's input column each key of the state names: the column of + // the key's lineage (a computed key only the column of the same group + // and column). let mapped = keys .iter() .map(|&key| { @@ -358,20 +364,19 @@ fn regrouped( }; column .filter(|&column| { - let (Some(at_state), Some(at_input)) = - (state.fields.get(key), rows.fields.get(column)) - else { - return false; - }; - at_state.is_plain() - && at_state.name == at_input.name - && at_state.table == at_input.table + state + .schema + .fields + .get(key) + .is_some_and(|field| field.is_plain()) + && matches!(state.column_lineage.get(key), + Some(lineage) if Some(lineage) == rows_lineage.get(column)) }) .ok_or(CoverageError::NotAGroupKey { key }) }) .collect::, _>>()?; let reduction = Reduction::by(mapped); - summary_agg_layout(&rows, family, &reduction).map_err(|error| { + summary_agg_layout(rows.schema(), family, &reduction).map_err(|error| { CoverageError::RegroupedBuild { reason: error.to_string(), } @@ -391,30 +396,35 @@ fn regrouped( }) } -/// The schema of the rows the build of summary state `state` reads (its +/// The input group the build of summary state `state` reads (its /// `SummaryAgg`'s input; one build over a chain's window reads its range of -/// the chain's input, which has the input's schema). `None` when `state` -/// is no summary state. -fn build_input_schema(state: &MemoGroupAsInput<'_>) -> Option { +/// the chain's input, which has the input's schema and lineage). `None` +/// when `state` is no summary state. +fn build_input<'m>(state: &MemoGroupAsInput<'m>) -> Option> { if let Some(states) = selected_from(state) { - return build_input_schema(&states); + return build_input(&states); } match state.operator() { Operator::ASAP( ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => Some(child.schema().clone()), - Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { - build_input_schema(children.first()?) - } + ) => Some(child), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => build_input(children.first()?), _ => None, } } /// The timestamp column of the rows the build of summary state `state` -/// reads ([`build_input_schema`]), as a selection names it. -fn time_column_of_build_input(state: &MemoGroupAsInput<'_>) -> Option { - let rows = build_input_schema(state)?; - column_identity(&rows, rows.time_index?) +/// reads ([`build_input`]), as a selection names it. +fn time_column_of_build_input(state: &MemoGroupAsInput<'_>) -> Option { + let rows = build_input(state)?; + let lineage: Vec> = rows + .group_properties() + .column_lineage + .iter() + .cloned() + .map(Some) + .collect(); + source_column(&lineage, rows.schema().time_index?) } /// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` @@ -453,7 +463,8 @@ fn of_summary_agg(build: &ASAPOp>) -> Result( child: &Built<'m>, filter: Option<&Predicate>>, @@ -486,7 +497,8 @@ fn of_aggregate<'m>( let InputOrTemporaryBuild::Input(base_input) = &base else { unreachable!("a rule builds only liftable operators over real inputs"); }; - let top = child.schema(); + let top = lineage_at_top(&base_input.group_properties().column_lineage, &chain); + let top = top.as_slice(); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. let mut lifted = BTreeMap::new(); @@ -533,7 +545,7 @@ fn of_aggregate<'m>( .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; let column = chain[depth].children()[0].schema().time_index?; let column = column_at_top(column, &chain[..=depth])?; - column_identity(top, column)?; + source_column(top, column)?; Some((column, combined_with(lifted.get(&column), constraint)?)) }); let lift_time = time.is_some(); @@ -566,7 +578,7 @@ fn of_aggregate<'m>( let selection = SelectionBox { columns: lifted .into_iter() - .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) + .filter_map(|(column, constraint)| Some((source_column(top, column)?, constraint))) .collect(), }; Ok(AggregateCoverage { @@ -579,13 +591,13 @@ fn of_aggregate<'m>( } /// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest, each scalar subquery +/// the agg child, whose lineage is `top`) and return the rest, each scalar subquery /// naming the input it reads. `above` /// are the operators between the predicate and the `SummaryAgg`, top to -/// bottom. A column [`column_identity`] cannot name stays. +/// bottom. A conjunct on a column [`source_column`] cannot name stays. fn residual( pred: &Predicate>, - top: &Schema, + top: &[Option], above: &[Operator>], lifted: &mut BTreeMap, ) -> Option>> { @@ -593,7 +605,7 @@ fn residual( for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; - column_identity(top, column)?; + source_column(top, column)?; Some((column, combined_with(lifted.get(&column), constraint)?)) }); match lift { @@ -649,19 +661,34 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option Option { - let field = schema.fields.get(column)?; - let namesakes = schema - .fields +/// The lineage of the aggregate's input columns: the lineage of `base` +/// (the group below the liftable `chain`, top down), carried up through +/// it: a `Filter`, `TimeRange` or `TimeShift` keeps its input's, and a +/// `Project`'s is [`project_lineage`]'s. `None` is a column computed in the +/// chain, which no selection names. +fn lineage_at_top(base: &[ColumnLineage], chain: &[Operator]) -> Vec> { + let mut lineage: Vec> = base.iter().cloned().map(Some).collect(); + for link in chain.iter().rev() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { + lineage = project_lineage(cols, &lineage); + } + } + lineage +} + +/// The source column a selection names column `column` by: its lineage, one +/// of `lineage` (per column; `None` computed). `None` when the column is +/// computed, or another column has the same lineage (`Project [a AS k, a AS +/// k2]`), so a selection cannot name it apart. +fn source_column(lineage: &[Option], column: ColumnId) -> Option { + let Some(ColumnLineage::SourceColumn(source_column)) = lineage.get(column)? else { + return None; + }; + let namesakes = lineage .iter() - .filter(|f| f.name == field.name && f.table == field.table) + .filter(|other| *other == &lineage[column]) .count(); - (namesakes == 1).then(|| ColumnIdentity { - table: field.table.clone(), - name: field.name.clone(), - }) + (namesakes == 1).then(|| source_column.clone()) } /// The interval on the timestamp column that `links` (top down, the diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index 23598bd52..fe500ae94 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -1,19 +1,27 @@ -//! Value sign: the sign of each output column of a memo group (concepts -//! doc, section 3, "Value sign"), derived once from the input groups' signs -//! and, for a `Scan`, the signs the data workload declares for its source's -//! columns. Error contracts that hold only for values of one sign read it. +//! Column lineage and value sign: two per-column group properties +//! (concepts doc, section 3, "Column lineage" and "Value sign"), derived +//! together, once per group, from the input groups' and, for a `Scan`, from +//! its source's columns and the signs the data workload declares for them. +//! One per-operator column layout says which input column each output +//! column copies (lineage and sign kept) and which it computes (lineage +//! `Computed` by this group, the sign by the operator's rule). Error +//! contracts that hold only for values of one sign read the sign; +//! aggregate coverage names a selection's columns by the lineage. use std::collections::BTreeMap; -use super::group::MemoGroupAsInput; -use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction}; +use super::group::{MemoGroupAsInput, MemoGroupId}; +use crate::ir::operator::arguments::{ + BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, Source, SourceColumn, +}; use crate::ir::operator::asap::{summary_agg_layout, summary_build}; use crate::ir::operator::input::OperatorInput; use crate::ir::operator::AggIntent; use crate::ir::scalar::expr_ir::{ArithmeticOpKind, ScalarValue}; use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; -use crate::ir::schema::{FieldDataType, Schema}; +use crate::ir::schema::topk::{ranked_row_projection, topk_output_name}; +use crate::ir::schema::{ColumnId, FieldDataType, Schema}; use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ ExactKind, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, @@ -42,71 +50,193 @@ pub enum ValueSign { Unknown, } +/// Where an output column's values come from, a group property derived +/// from the input groups (concepts doc, section 3, "Column lineage"). It +/// names a column across groups where a schema field's name cannot: two +/// join sides' columns of one name have different lineage, a self-join's +/// two scans of one source have different scan ordinals, and a renamed +/// column keeps its lineage. A column passed through unchanged keeps its +/// input column's lineage, so two columns have equal lineage only when +/// they hold the same values: the same column of the same scan, or the +/// same computation. Aggregate coverage names a selection's column by its +/// source column. +#[derive(Debug, Clone, PartialEq)] +pub enum ColumnLineage { + /// Copied unchanged from a column of one scan of a source. + SourceColumn(SourceColumn), + /// Column `column` of memo group `group`, where it was computed: a + /// measure, an expression, a summary state or a readout, or a column + /// the inputs of a union-like operator (`SetOp`, `Concat`, + /// `SummaryMerge`) disagree on. + Computed { + group: MemoGroupId, + column: ColumnId, + }, +} + use ValueSign::{NonNegative, NonPositive, Unknown}; type Input<'m> = MemoGroupAsInput<'m>; -/// The sign of each column of `output`, the output schema of `operator`, -/// from its input groups' `column_signs` and, for a `Scan`, `source_facts`. -pub(super) fn column_signs( +/// One output column's lineage and sign. +#[derive(Clone)] +struct ColumnProperties { + /// `None` for a column the group being derived computes: its lineage + /// is `Computed` by that group at the column's final position. + lineage: Option, + sign: ValueSign, +} + +/// A column the operator computes, with no sign known. +impl Default for ColumnProperties { + fn default() -> Self { + Self::computed(Unknown) + } +} + +/// `columns` columns the operator computes, with no sign known. +fn all_computed(columns: usize) -> Vec { + vec![ColumnProperties::default(); columns] +} + +impl ColumnProperties { + /// A column the operator computes, with values of `sign`. + fn computed(sign: ValueSign) -> Self { + Self { + lineage: None, + sign, + } + } + + /// What two inputs' columns at one position agree on: each property + /// where they are equal, else computed by this group / `Unknown`. + fn agreed_with(self, other: &Self) -> Self { + Self { + lineage: self + .lineage + .filter(|lineage| other.lineage.as_ref() == Some(lineage)), + sign: agree(self.sign, other.sign), + } + } +} + +/// The lineage and sign of each column of `output`, the output schema of +/// `operator` computed by memo group `group`, from its input groups' +/// `column_lineage` and `column_signs` and, for a `Scan`, its source and +/// `source_facts`. +pub(super) fn column_lineage_and_signs( + group: MemoGroupId, operator: &Operator>, output: &Schema, source_facts: &BTreeMap, -) -> Vec { - let signs = match operator { - Operator::NonASAP(op) => non_asap_signs(op, output, source_facts), - Operator::ASAP(op) => asap_signs(op, output), +) -> (Vec, Vec) { + let columns = match operator { + Operator::NonASAP(op) => non_asap_columns(op, output, source_facts), + Operator::ASAP(op) => asap_columns(op, output), }; debug_assert_eq!( - signs.len(), + columns.len(), output.fields.len(), - "one sign per output column of {}", + "one lineage and sign per output column of {}", operator.kind_name() ); - signs + columns + .into_iter() + .enumerate() + .map(|(index, properties)| { + let lineage = properties.lineage.unwrap_or(ColumnLineage::Computed { + group, + column: index, + }); + (lineage, properties.sign) + }) + .unzip() } -fn signs_of(input: &Input<'_>) -> Vec { - input.group_properties().column_signs.clone() +/// The columns of a memo group, read from its `group_properties`. +fn columns_of(input: &Input<'_>) -> Vec { + let properties = input.group_properties(); + properties + .column_lineage + .iter() + .zip(&properties.column_signs) + .map(|(lineage, &sign)| ColumnProperties { + lineage: Some(lineage.clone()), + sign, + }) + .collect() } -fn non_asap_signs( +/// `right`, the right side's columns of a `Join`, which concatenates them +/// after `left`'s: a source column of a source `left` also +/// scans moves to the scans after `left`'s (its scan ordinal shifted by +/// `left`'s largest for that source, plus one), so a self-join's two sides +/// name different scans. +fn after_left_scans( + left: &[ColumnProperties], + mut right: Vec, +) -> Vec { + let mut first_free_scan: BTreeMap<&Source, u32> = BTreeMap::new(); + for column in left { + if let Some(ColumnLineage::SourceColumn(source_column)) = &column.lineage { + let free = first_free_scan.entry(&source_column.source).or_default(); + *free = (*free).max(source_column.scan_ordinal + 1); + } + } + for column in &mut right { + if let Some(ColumnLineage::SourceColumn(source_column)) = &mut column.lineage { + if let Some(shift) = first_free_scan.get(&source_column.source) { + source_column.scan_ordinal += shift; + } + } + } + right +} + +fn non_asap_columns( op: &NonASAPOp>, output: &Schema, source_facts: &BTreeMap, -) -> Vec { +) -> Vec { use NonASAPOp::*; - let unknown = || vec![Unknown; output.fields.len()]; match op { - // The signs its source's facts declare (a histogram's or summary's - // series reads its family's); a counter's `value` is non-negative - // without a declaration. + // Each column copied from its source, with the sign its source's + // facts declare (a histogram's or summary's series reads its + // family's); a counter's `value` is non-negative without a + // declaration. Scan { source, schema, .. } => { let facts = series_facts(source_facts, source_name(source)); schema .fields .iter() - .map(|field| match facts { - Some((facts, series_type)) => facts - .column_signs - .get(&field.name) - .copied() - .or((field.name == "value" - && series_type == Some(PrometheusMetricType::Counter)) - .then_some(NonNegative)) - .unwrap_or(Unknown), - None => Unknown, + .map(|field| ColumnProperties { + sign: match facts { + Some((facts, series_type)) => facts + .column_signs + .get(&field.name) + .copied() + .or((field.name == "value" + && series_type == Some(PrometheusMetricType::Counter)) + .then_some(NonNegative)) + .unwrap_or(Unknown), + None => Unknown, + }, + lineage: Some(ColumnLineage::SourceColumn(SourceColumn { + source: source.clone(), + scan_ordinal: 0, + column: field.name.clone(), + })), }) .collect() } - Values { .. } => unknown(), + Values { .. } => all_computed(output.fields.len()), // The scalar's value, every other column `Unknown`. PromqlVectorFromScalar(expr) => { - let mut signs = unknown(); + let mut columns = all_computed(output.fields.len()); if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, output) { - signs[value] = expression_sign(expr, &[]); + columns[value] = ColumnProperties::computed(expression_sign(expr, &[])); } - signs + columns } Filter { child, .. } | Sort { child, .. } @@ -116,76 +246,142 @@ fn non_asap_signs( | TimeShift { child, .. } | PromqlSubquery { child, .. } | PromqlSeriesSample { child, .. } - | PromqlInfoEnrich { child, .. } => signs_of(child), + | PromqlInfoEnrich { child, .. } => columns_of(child), // The relabeled column is written; every other column is copied. PromqlRelabel { dst, child, .. } => { - let mut signs = signs_of(child); + let mut columns = columns_of(child); + let written = ColumnProperties::default(); match child.schema().column_id(dst) { - Some(written) => signs[written] = Unknown, - None => signs.push(Unknown), + Some(column) => columns[column] = written, + None => columns.push(written), } - signs + columns } - Project { cols, child, .. } => project_signs(cols, &signs_of(child)), + Project { cols, child, .. } => project_columns(cols, child), Aggregate { reduction, measures, + output_names, child, .. } => { - if let [AggIntent::TopK { .. }] = measures.as_slice() { + if let [AggIntent::TopK { k, .. }] = measures.as_slice() { // The ranked rows, projected: each column copied. - return match crate::ir::schema::topk::ranked_row_projection(child, reduction, "") { - Ok(cols) => project_signs(&cols, &signs_of(child)), - Err(_) => unknown(), + let name = topk_output_name(*k, output_names); + return match ranked_row_projection(child, reduction, &name) { + Ok(cols) => project_columns(&cols, child), + Err(_) => all_computed(output.fields.len()), }; } - aggregate_signs(child, reduction, measures, output) + aggregate_columns(child, reduction, measures, output) } Join { kind, left, right, .. } => { - let mut signs = signs_of(left); + let mut columns = columns_of(left); if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { - signs.extend(signs_of(right)); + let right = after_left_scans(&columns, columns_of(right)); + columns.extend(right); } - signs + columns } - SetOp { left, right, .. } => agreed(&[signs_of(left), signs_of(right)]), - Concat { children } => agreed(&children.iter().map(signs_of).collect::>()), + SetOp { left, right, .. } => agreed(&[columns_of(left), columns_of(right)]), + Concat { children } => agreed(&children.iter().map(columns_of).collect::>()), SQLWindowFunc { child, .. } => { - let mut signs = signs_of(child); - signs.push(Unknown); - signs + let mut columns = columns_of(child); + columns.push(ColumnProperties::default()); + columns } - // The left side's columns, its value per the operator, then the - // labels the match adds (unknown). + // The output side's columns (the left's; the right's, the "many" + // side, for `group_right`), its value computed per the operator, + // then the right side's labels the match adds. A matched label has + // equal values on both sides, so it takes the output side's lineage + // and sign; `or`, a union, keeps what both sides agree on (a label + // one side lacks is computed). A label + // `group_left(...)` names comes from the right ("one") side and one + // `group_right(...)` names from the left, overwriting a column of + // that name. The output does not concatenate both sides' columns, + // so no scan ordinal shifts. BinaryOp { operator, lhs, rhs, .. } => { - let mut signs = signs_of(lhs); + let left = columns_of(lhs); + let right = columns_of(rhs); + let named = |input: &Input<'_>, columns: &[ColumnProperties], name: &str| { + input + .schema() + .column_id(name) + .and_then(|column| columns.get(column).cloned()) + .unwrap_or_default() + }; + let grouping = operator + .vector_match + .as_ref() + .and_then(|m| m.grouping.as_ref()); + let output_side_is_right = + grouping.is_some_and(|grouping| grouping.side == GroupSide::Right); + let is_or = matches!(operator.kind, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)); + let mut columns: Vec = lhs + .schema() + .fields + .iter() + .enumerate() + .map(|(column, field)| { + let from_right = || named(rhs, &right, &field.name); + if output_side_is_right { + from_right() + } else if is_or { + left[column].clone().agreed_with(&from_right()) + } else { + left[column].clone() + } + }) + .collect(); if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { - let rhs_value = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) + let left_sign = left[value].sign; + let right_sign = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) .ok() - .map_or(Unknown, |column| signs_of(rhs)[column]); - signs[value] = match &operator.kind { + .and_then(|column| right.get(column)) + .map_or(Unknown, |column| column.sign); + columns[value] = ColumnProperties::computed(match &operator.kind { // A filter keeps the left side's values. BinaryOpKind::Compare(_) | BinaryOpKind::Set( PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless, - ) => signs[value], - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(signs[value], rhs_value), + ) => left_sign, + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(left_sign, right_sign), BinaryOpKind::CompareBool(_) => NonNegative, - BinaryOpKind::Arithmetic(op) => arithmetic_sign(op, signs[value], rhs_value), + BinaryOpKind::Arithmetic(op) => arithmetic_sign(op, left_sign, right_sign), + }); + } + // Under `or` a label only the right side has is null on the left + // side's rows: computed, like a label only the left side has. + let added = output.fields.get(columns.len()..).unwrap_or_default(); + columns.extend(added.iter().map(|field| { + if is_or { + ColumnProperties::default() + } else { + named(rhs, &right, &field.name) + } + })); + columns.truncate(output.fields.len()); + if let Some(grouping) = grouping { + let (one_side, one_side_columns) = match grouping.side { + GroupSide::Left => (rhs, &right), + GroupSide::Right => (lhs, &left), }; + for label in &grouping.labels { + if let Some(column) = output.column_id(label) { + columns[column] = named(one_side, one_side_columns, label); + } + } } - signs.resize(output.fields.len(), Unknown); - signs + columns } } } -fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { +fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec { use ASAPOp::*; match op { SummaryAgg { @@ -194,8 +390,8 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { input, reduction, .. - } => state_signs(child, summary_type, input, reduction) - .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs), + } => state_columns(child, summary_type, input, reduction) + .map_or_else(|| all_computed(output.fields.len()), |(_, columns)| columns), // The build's layout without its own time column, after the // computed `window_end` (column 0). SummaryTumblingWindowChain { @@ -204,15 +400,15 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { input, reduction, .. - } => match state_signs(child, summary_type, input, reduction) { - Some((build, mut signs)) => { + } => match state_columns(child, summary_type, input, reduction) { + Some((build, mut columns)) => { if let Some(time) = build.time_index { - signs.remove(time); + columns.remove(time); } - signs.insert(0, Unknown); - signs + columns.insert(0, ColumnProperties::default()); + columns } - None => vec![Unknown; output.fields.len()], + None => all_computed(output.fields.len()), }, SummaryMerge { children, group_by } => { // The windows a `TimeRange` over a chain selects merge into the @@ -226,22 +422,26 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { .. }) = summary_build(child) { - return state_signs(&child, &summary_type, &input, &reduction) - .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs); + return state_columns(&child, &summary_type, &input, &reduction) + .map_or_else(|| all_computed(output.fields.len()), |(_, columns)| columns); } } - let merged = agreed(&children.iter().map(signs_of).collect::>()); + let merged = agreed(&children.iter().map(columns_of).collect::>()); match group_by { None => merged, + // The keys copied; the state, merged across the inputs' + // groups, computed with the sign they agree on. Some(keys) => { + let column_at = |column: usize| merged.get(column).cloned().unwrap_or_default(); let state = children[0] .schema() .fields .iter() - .position(|field| !field.is_plain()); + .position(|field| !field.is_plain()) + .map(|state| ColumnProperties::computed(column_at(state).sign)); keys.iter() - .chain(state.as_ref()) - .map(|&column| merged.get(column).copied().unwrap_or(Unknown)) + .map(|&key| column_at(key)) + .chain(state) .collect() } } @@ -249,7 +449,7 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { SummaryEstimate { summary_input, query, - } => readout_signs(summary_input, output, |state, summary_type| match query { + } => readout_columns(summary_input, output, |state, summary_type| match query { SketchStatistic::Cardinality | SketchStatistic::FrequencyL2 | SketchStatistic::FrequencyEntropy => NonNegative, @@ -270,27 +470,51 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { FieldDataType::ExactAggregate(ExactKind::Count, _) ) }); - readout_signs( + readout_columns( child, output, |state, _| if counts { NonNegative } else { state }, ) } - MaintainPopulation { child, .. } => signs_of(child), - EvaluatePopulation { child, .. } if output == child.schema() => signs_of(child), + MaintainPopulation { child, .. } => columns_of(child), + EvaluatePopulation { child, .. } if output == child.schema() => columns_of(child), EvaluatePopulation { .. } | SummarySubtract { .. } | SummaryDelete { .. } | SummaryJoin { .. } - | Extension { .. } => vec![Unknown; output.fields.len()], + | Extension { .. } => all_computed(output.fields.len()), } } -/// A column item keeps its input column's sign; an expression item's sign -/// is [`expression_sign`]'s. -fn project_signs(cols: &[ProjectItem], input: &[ValueSign]) -> Vec { +/// The columns of `cols`, a `Project` over `child`: lineage by +/// [`project_lineage`], sign by [`expression_sign`]. +fn project_columns(cols: &[ProjectItem], child: &Input<'_>) -> Vec { + let input = child.group_properties(); + let input_lineage: Vec> = + input.column_lineage.iter().cloned().map(Some).collect(); + project_lineage(cols, &input_lineage) + .into_iter() + .zip(cols) + .map(|(lineage, item)| ColumnProperties { + lineage, + sign: expression_sign(&item.expr, &input.column_signs), + }) + .collect() +} + +/// The lineage of the output columns of `cols`, a `Project` over input +/// columns of lineage `input` (`None`: computed by the group being +/// derived): a column item, renamed or not, copies its input column's; an +/// expression item is `None`, computed by the `Project`'s group. +pub(super) fn project_lineage( + cols: &[ProjectItem], + input: &[Option], +) -> Vec> { cols.iter() - .map(|item| expression_sign(&item.expr, input)) + .map(|item| match item.expr { + ScalarExpr::Column(column) => input.get(column).cloned().flatten(), + _ => None, + }) .collect() } @@ -322,46 +546,46 @@ fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { } /// An aggregate's, by where each output column comes from -/// ([`aggregate_output`]): a kept input column keeps its sign, a -/// measure's value has [`measure_sign`] of its input column's, and a label -/// a measure adds is `Unknown`. -fn aggregate_signs( +/// ([`aggregate_output`]): a kept input column is copied, a measure's +/// value is computed with [`measure_sign`] of its input column's sign, and +/// a label a measure adds is computed with no sign. +fn aggregate_columns( child: &Input<'_>, reduction: &Reduction, measures: &[AggIntent], output: &Schema, -) -> Vec { +) -> Vec { let Ok((_, sources)) = aggregate_output(child.schema(), reduction, measures, &[]) else { - return vec![Unknown; output.fields.len()]; + return all_computed(output.fields.len()); }; - signs_from_sources(child, &sources, |index, input| { + columns_from_sources(child, &sources, |index, input| { measure_sign(&measures[index], input) }) } -/// The sign of each output column from `sources`: an input column keeps -/// its sign, measure `index` has `measure(index, the sign of its input -/// column)`, and an added label is `Unknown`. -fn signs_from_sources( +/// Each output column from `sources`: an input column is copied, measure +/// `index` is computed with `measure(index, the sign of its input +/// column)`, and an added label is computed with no sign. +fn columns_from_sources( child: &Input<'_>, sources: &[OutputColumnSource], measure: impl Fn(usize, ValueSign) -> ValueSign, -) -> Vec { - let input = &child.group_properties().column_signs; - let sign_of = |column: Option| { +) -> Vec { + let input = columns_of(child); + let column_at = |column: Option| { column - .and_then(|column| input.get(column).copied()) - .unwrap_or(Unknown) + .and_then(|column| input.get(column).cloned()) + .unwrap_or_default() }; sources .iter() .map(|source| match source { - OutputColumnSource::InputColumn(column) => sign_of(Some(*column)), + OutputColumnSource::InputColumn(column) => column_at(Some(*column)), OutputColumnSource::Measure { index, input_column, - } => measure(*index, sign_of(*input_column)), - OutputColumnSource::AddedLabel => Unknown, + } => ColumnProperties::computed(measure(*index, column_at(*input_column).sign)), + OutputColumnSource::AddedLabel => ColumnProperties::default(), }) .collect() } @@ -393,51 +617,50 @@ fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { } } -/// A summary state's schema and signs, in its layout -/// ([`summary_agg_layout`]): its key columns keep their input's sign and its -/// state column records the sign of the values it is updated with; `None` -/// when the layout does not derive. -fn state_signs( +/// A summary state's schema and columns, in its layout +/// ([`summary_agg_layout`]): its key columns copy their input's and its +/// state column is computed and records the sign of the values it is +/// updated with; `None` when the layout does not derive. +fn state_columns( child: &Input<'_>, summary_type: &FieldDataType, update: &SummaryUpdate, reduction: &Reduction, -) -> Option<(Schema, Vec)> { +) -> Option<(Schema, Vec)> { let (schema, sources) = summary_agg_layout(child.schema(), summary_type, reduction).ok()?; let updated_with = match &update.weight { SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) .ok() - .and_then(|column| signs_of(child).get(column).copied()) + .and_then(|column| child.group_properties().column_signs.get(column).copied()) .unwrap_or(Unknown), SummaryInputExpr::Constant(value) => sign_of_number(*value), SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, }; - let signs = signs_from_sources(child, &sources, |_, _| updated_with); - Some((schema, signs)) + let columns = columns_from_sources(child, &sources, |_, _| updated_with); + Some((schema, columns)) } -/// A readout of summary state: the state's plain columns keep their sign -/// in place, and the value read out has `value(the state's recorded sign, -/// the state column's type)`. -/// `Unknown` everywhere when the output does not keep the state's layout (a -/// top-k's rows). -fn readout_signs( +/// A readout of summary state: the state's plain columns are copied in +/// place, and the value read out is computed, with `value(the state's +/// recorded sign, the state column's type)`. All computed with no sign +/// when the output does not keep the state's layout (a top-k's rows). +fn readout_columns( state: &Input<'_>, output: &Schema, value: impl Fn(ValueSign, &FieldDataType) -> ValueSign, -) -> Vec { +) -> Vec { let input = state.schema(); if input.fields.len() != output.fields.len() { - return vec![Unknown; output.fields.len()]; + return all_computed(output.fields.len()); } - signs_of(state) + columns_of(state) .into_iter() .zip(&input.fields) - .map(|(sign, field)| { + .map(|(column, field)| { if field.is_plain() { - sign + column } else { - value(sign, &field.dtype) + ColumnProperties::computed(value(column.sign, &field.dtype)) } }) .collect() @@ -470,17 +693,17 @@ fn quantile_sign(q: f64, input: ValueSign) -> ValueSign { } } -/// Per column, the inputs' sign if they all agree, else `Unknown`. -fn agreed(inputs: &[Vec]) -> Vec { +/// Per column, what all inputs agree on ([`ColumnProperties::agreed_with`]). +fn agreed(inputs: &[Vec]) -> Vec { let Some((first, rest)) = inputs.split_first() else { return Vec::new(); }; first .iter() .enumerate() - .map(|(column, &sign)| { - rest.iter().fold(sign, |agreed, other| { - agree(agreed, other.get(column).copied().unwrap_or(Unknown)) + .map(|(column, properties)| { + rest.iter().fold(properties.clone(), |agreed, other| { + agreed.agreed_with(&other.get(column).cloned().unwrap_or_default()) }) }) .collect() diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 474c4dd13..4fdde2aaf 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -4,7 +4,7 @@ use std::collections::BTreeSet; use super::aggregate_coverage::{aggregate_coverage, AggregateCoverage, CoverageError}; -use super::column_properties::{column_signs, ValueSign}; +use super::column_properties::{column_lineage_and_signs, ColumnLineage, ValueSign}; use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; use crate::ir::operator::input::OperatorInput; use crate::ir::schema::Schema; @@ -60,8 +60,14 @@ impl MemoGroup { /// operator nodes. #[derive(Debug, Clone)] pub struct GroupProperties { + /// Fields: name, type, nullability. pub schema: Schema, pub result_kind: OperatorResultKind, + /// Per output column, in schema order: the source column (of one scan) + /// its values are copied from, or the group and column where they were + /// computed (concepts doc, section 3, "Column lineage"), from the input + /// groups' and, for a `Scan`, its source's columns. + pub column_lineage: Vec, /// Per output column, in schema order: the sign of its values, from the /// input groups' and, for a `Scan`, the declared ones. pub column_signs: Vec, @@ -117,19 +123,23 @@ pub(super) fn validate_expression_over_input_groups( Ok((schema, merge_coverage)) } -/// The properties of a group computing `expression` over its input -/// groups, once it is valid over them +/// The properties of new memo group `group` computing `expression` over +/// its input groups, once it is valid over them /// ([`validate_expression_over_input_groups`]), reading each input group's /// `group_properties` and `expression`. pub(super) fn derive_group_properties( memo: &Memo, + group: MemoGroupId, expression: &Operator, ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); let (schema, merge_coverage) = validate_expression_over_input_groups(&operator)?; let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(&operator).ok()); + let (column_lineage, column_signs) = + column_lineage_and_signs(group, &operator, &schema, &memo.source_facts); Ok(GroupProperties { - column_signs: column_signs(&operator, &schema, &memo.source_facts), + column_lineage, + column_signs, depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 1bbafc4ec..69443a586 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -124,11 +124,11 @@ impl Memo { { return Ok(*group); } - let group_properties = derive_group_properties(self, &expression)?; - let shareable = group_properties.shareable_when_equal; let id = MemoGroupId( u32::try_from(self.groups.len()).expect("a memo has at most u32::MAX groups"), ); + let group_properties = derive_group_properties(self, id, &expression)?; + let shareable = group_properties.shareable_when_equal; for &input in expression.children() { add_parent(&mut self.groups[input.0 as usize], id); } diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index e64e092c5..69f81a7b1 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -32,7 +32,6 @@ fn project_first_column(input: MemoGroupId) -> Operator { alias: None, expr: crate::ir::ScalarExpr::Column(0), }], - qualifier: None, child: input, }) } @@ -562,7 +561,6 @@ fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { alias: Some("v".into()), expr: ScalarExpr::Literal(ScalarValue::Float64(value)), }], - qualifier: None, child, }) }) diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 9e2b781cc..0658f80a5 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -16,7 +16,7 @@ pub use candidate::{ CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, }; -pub use column_properties::ValueSign; +pub use column_properties::{ColumnLineage, ValueSign}; pub use edge::LogicalEdge; pub use group::{GroupProperties, MemoGroup, MemoGroupError, MemoGroupId}; pub use memo::{Memo, MemoQuery}; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 3bfd01e38..8a6bdd5e2 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -14,7 +14,7 @@ mod pruning; pub use logical::{ view_coverage, AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, - CandidateForMemoGroup, CandidateForMemoGroupId, CoverageError, GroupProperties, + CandidateForMemoGroup, CandidateForMemoGroupId, ColumnLineage, CoverageError, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupError, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 60833bac8..d55290ee6 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -84,16 +84,10 @@ pub fn population_over_selector( ) -> Rc { use crate::ir::operator::arguments::Source; use crate::ir::operator::maintained_population::*; - if let Some(NonASAPOp::Project { - cols, - qualifier, - child, - }) = root.non_asap() - { + if let Some(NonASAPOp::Project { cols, child }) = root.non_asap() { let child = population_over_selector(child, statistic.clone(), grouping); let project = Operator::NonASAP(NonASAPOp::Project { cols: cols.clone(), - qualifier: qualifier.clone(), child, }); return Rc::new(OperatorNode::with_schema(project, root.schema.clone())); diff --git a/crates/types/src/test_support/memo_fixtures.rs b/crates/types/src/test_support/memo_fixtures.rs index 6fc9b4d40..5da450b33 100644 --- a/crates/types/src/test_support/memo_fixtures.rs +++ b/crates/types/src/test_support/memo_fixtures.rs @@ -3,7 +3,7 @@ //! `memo_aggregate_coverage`) build their examples from. use std::time::Duration; -use crate::ir::operator::arguments::{AtModifier, Reduction, Source, TimeShift}; +use crate::ir::operator::arguments::{AtModifier, Reduction, Source, SourceColumn, TimeShift}; use crate::ir::operator::AggIntent; use crate::ir::scalar::ColumnRef; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; @@ -37,6 +37,28 @@ pub fn scan( }) } +/// Column `column` of scan `scan_ordinal` of the series `metric`. +pub fn series_column(metric: &str, scan_ordinal: u32, column: &str) -> SourceColumn { + SourceColumn { + source: Source::TimeSeries { + metric: metric.into(), + }, + scan_ordinal, + column: column.into(), + } +} + +/// Column `column` of scan `scan_ordinal` of the table `table`. +pub fn table_column(table: &str, scan_ordinal: u32, column: &str) -> SourceColumn { + SourceColumn { + source: Source::Table { + table_ref: table.into(), + }, + scan_ordinal, + column: column.into(), + } +} + pub fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { Operator::NonASAP(NonASAPOp::Filter { pred: Predicate(pred), diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs new file mode 100644 index 000000000..94bc495db --- /dev/null +++ b/crates/types/tests/column_lineage.rs @@ -0,0 +1,730 @@ +//! Column lineage, derived once per memo group from its input groups' +//! (concepts doc `memo-groups-properties-requirements.md`, section 3, +//! "Column lineage"): the doc's examples 1 to 3. Example 4 (PromQL label +//! matchers) is `scan_predicates_lift_and_their_states_merge` in +//! `memo_aggregate_coverage.rs`. Then a self-join's two scans, and the +//! lineage and sign of PromQL match-grouping labels. +use std::collections::BTreeMap; + +use asap_types::ir::operator::arguments::{ + AtModifier, BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, Source, + SourceColumn, VectorGrouping, VectorMatch, VectorMatchKind, +}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::selection::Constraint; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, ScalarExpr, +}; +use asap_types::stages::{ + CandidateExpression, CandidateForMemoGroup, ColumnLineage, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, ValueSign, +}; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::test_support::memo_fixtures::{ + self, aggregate, filter, range, series_column, shift, table_column, +}; +use asap_types::workload::{DataWorkload, SourceFacts}; + +fn memo() -> Memo { + Memo::new(&DataWorkload::default()) +} + +/// `Scan table` of Utf8 `columns`. +fn scan(table: &str, columns: &[&str]) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: table.into(), + }, + predicates: vec![], + schema: Schema::new( + columns + .iter() + .map(|name| Field::plain(*name, DataType::Utf8, false)) + .collect(), + ), + }) +} + +/// `column = value`. +fn equals(column: usize, value: &str) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(value.into()))), + semantics: ExprSemantics::Sql, + } +} + +/// `SummaryAgg[HLL](by keys; column)`: the distinct values of `column`. +fn hll(child: MemoGroupId, keys: Vec, column: &str) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Hll), + input: SummaryUpdate::column(ColumnRef::Named(column.into())), + reduction: Reduction::by(keys), + filter: None, + }) +} + +/// Column `column` of memo group `group`, where it was computed. +fn computed(group: MemoGroupId, column: usize) -> ColumnLineage { + ColumnLineage::Computed { group, column } +} + +fn lineage(memo: &Memo, group: MemoGroupId) -> &[ColumnLineage] { + &memo.groups()[group.0 as usize] + .group_properties + .column_lineage +} + +/// Example 1: `SELECT f.src, approx_distinct(f.dst) FROM flows f WHERE +/// f.proto = 'tcp' GROUP BY f.src`. The group-by key keeps `flows.src` +/// through the aggregate, the HLL state and its readout; the readout +/// renamed to the aggregate's column name has the aggregate's schema, so +/// `add_candidate` accepts it. +#[test] +fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { + let mut memo = memo(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst", "proto", "ts"])) + .unwrap(); + let g2 = memo + .find_or_add_group(filter(equals(2, "tcp"), g1)) + .unwrap(); + let g3 = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Cardinality { cols: vec![1] }], + output_names: vec!["approx_distinct(dst)".into()], + filters: vec![], + having: None, + child: g2, + })) + .unwrap(); + let g4 = memo.find_or_add_group(hll(g2, vec![0], "dst")).unwrap(); + let g5 = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Cardinality, + })) + .unwrap(); + let flows = ["src", "dst", "proto", "ts"] + .map(|column| ColumnLineage::SourceColumn(table_column("flows", 0, column))); + assert_eq!(lineage(&memo, g1), flows); + assert_eq!(lineage(&memo, g2), flows); + // Each of the aggregate, the state and the readout computes its own + // second column. + for group in [g3, g4, g5] { + assert_eq!( + lineage(&memo, group), + [ + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + computed(group, 1) + ] + ); + } + // The state's selection names the filtered column by its lineage. + let coverage = memo.groups()[g4.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert_eq!( + coverage.selection()[0].columns()[&table_column("flows", 0, "proto")], + Constraint::In(vec![ScalarValue::Utf8("tcp".into())]) + ); + + let operator = Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: Some("src".into()), + expr: ScalarExpr::Column(0), + }, + ProjectItem { + alias: Some("approx_distinct(dst)".into()), + expr: ScalarExpr::Column(1), + }, + ], + child: g5, + }); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + let id = memo.add_candidate(g3, candidate).unwrap(); + assert_eq!(id.group, g3); +} + +/// Example 2: `SELECT source, count(*) FROM (SELECT f.src AS source FROM +/// flows f) GROUP BY source`. The renaming `Project` keeps `flows.src`; a +/// state grouped by `source` has the key lineage of one grouped by `src` +/// over the scan, a selection on the renamed `area` names `flows.region`, +/// and the rollup's group-key check, which compares lineage, passes. +#[test] +fn a_rename_keeps_lineage_and_the_rollup_key_check_compares_it() { + let mut memo = memo(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst", "region"])) + .unwrap(); + let renamed = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: [("source", 0), ("dst", 1), ("area", 2)] + .map(|(alias, column)| ProjectItem { + alias: Some(alias.into()), + expr: ScalarExpr::Column(column), + }) + .to_vec(), + child: g1, + })) + .unwrap(); + assert_eq!( + lineage(&memo, renamed), + [ + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + ColumnLineage::SourceColumn(table_column("flows", 0, "dst")), + ColumnLineage::SourceColumn(table_column("flows", 0, "region")), + ] + ); + + let by_source = memo + .find_or_add_group(hll(renamed, vec![0], "dst")) + .unwrap(); + let by_src = memo.find_or_add_group(hll(g1, vec![0], "dst")).unwrap(); + assert_eq!(lineage(&memo, by_source)[0], lineage(&memo, by_src)[0]); + + let in_us = memo + .find_or_add_group(filter(equals(2, "us"), renamed)) + .unwrap(); + let by_source_and_area = memo + .find_or_add_group(hll(in_us, vec![0, 2], "dst")) + .unwrap(); + let coverage = memo.groups()[by_source_and_area.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert_eq!( + coverage.selection()[0].columns()[&table_column("flows", 0, "region")], + Constraint::In(vec![ScalarValue::Utf8("us".into())]) + ); + + let rolled_up_id = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![by_source_and_area], + group_by: Some(vec![0]), + })) + .unwrap(); + let rolled_up = &memo.groups()[rolled_up_id.0 as usize].group_properties; + // The regrouped state is computed by the merge, not its input's. + assert_eq!( + rolled_up.column_lineage, + [ + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + computed(rolled_up_id, 1) + ] + ); + assert!(rolled_up.aggregate_coverage.is_some()); +} + +/// Example 3: `SELECT s.region, count(*) FROM sales s JOIN users u ON +/// s.user = u.id GROUP BY s.region`. Both join sides have a `region` +/// column; their lineage tells them apart, so selections on them are +/// different columns. +#[test] +fn join_columns_of_one_name_have_distinct_lineage() { + let mut memo = memo(); + let sales = memo + .find_or_add_group(scan("sales", &["user", "region"])) + .unwrap(); + let users = memo + .find_or_add_group(scan("users", &["id", "region"])) + .unwrap(); + let joined = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(2)), + semantics: ExprSemantics::Sql, + }), + left: sales, + right: users, + })) + .unwrap(); + assert_eq!( + lineage(&memo, joined), + [ + ColumnLineage::SourceColumn(table_column("sales", 0, "user")), + ColumnLineage::SourceColumn(table_column("sales", 0, "region")), + ColumnLineage::SourceColumn(table_column("users", 0, "id")), + ColumnLineage::SourceColumn(table_column("users", 0, "region")), + ] + ); + + let selected_column = |memo: &mut Memo, region_column: usize| { + let in_us = memo + .find_or_add_group(filter(equals(region_column, "us"), joined)) + .unwrap(); + let state = memo.find_or_add_group(hll(in_us, vec![], "id")).unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + selection.columns().keys().cloned().collect::>() + }; + assert_eq!( + selected_column(&mut memo, 1), + [table_column("sales", 0, "region")] + ); + assert_eq!( + selected_column(&mut memo, 3), + [table_column("users", 0, "region")] + ); +} + +/// A predicate on a computed column is not lifted: no selection can name +/// the column, so it stays in the definition's unlifted operators. +#[test] +fn a_predicate_on_a_computed_column_stays_unlifted() { + let mut memo = memo(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst"])) + .unwrap(); + let tagged_rows = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: Some("src".into()), + expr: ScalarExpr::Column(0), + }, + ProjectItem { + alias: Some("tag".into()), + expr: ScalarExpr::Literal(ScalarValue::Utf8("x".into())), + }, + ], + child: g1, + })) + .unwrap(); + assert_eq!( + lineage(&memo, tagged_rows), + [ + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + computed(tagged_rows, 1) + ] + ); + let tagged = memo + .find_or_add_group(filter(equals(1, "x"), tagged_rows)) + .unwrap(); + let state = memo.find_or_add_group(hll(tagged, vec![], "src")).unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert!(coverage.selection()[0].columns().is_empty()); + assert!(coverage + .definition() + .unlifted_operators + .contains(&filter(equals(1, "x"), MemoGroupId(0)).map_children(|_| None))); +} + +/// `Scan latency`: `[ts, value, job]`, `ts` its time index. +fn latency_series() -> Operator { + memo_fixtures::scan("latency", &[("job", DataType::Utf8)], vec![]) +} + +/// A chain of `SummaryAgg[KLL](by job; value)` windows: `[window_end, job, +/// state]`. `window_end` (column 0) and the state are computed and the key +/// keeps its lineage. A `TimeShift` and a `TimeRange` over the chain select +/// windows, so they pass the chain's lineage through; the merge of the +/// selected windows has the layout of the chain's build, `[job, state]`. +#[test] +fn a_chain_computes_window_end_and_state_and_its_window_selections_pass_lineage_through() { + let mut memo = memo(); + let series = memo.find_or_add_group(latency_series()).unwrap(); + let chain = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: series, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + tumbling_window_width_ms: 60_000, + })) + .unwrap(); + let chain_lineage = [ + computed(chain, 0), + ColumnLineage::SourceColumn(series_column("latency", 0, "job")), + computed(chain, 2), + ]; + assert_eq!(lineage(&memo, chain), chain_lineage); + + let an_hour_ago = memo + .find_or_add_group(shift(3_600_000, None, chain)) + .unwrap(); + assert_eq!(lineage(&memo, an_hour_ago), chain_lineage); + let five_minutes_an_hour_ago = memo.find_or_add_group(range(300_000, an_hour_ago)).unwrap(); + assert_eq!(lineage(&memo, five_minutes_an_hour_ago), chain_lineage); + + let merged = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![five_minutes_an_hour_ago], + group_by: None, + })) + .unwrap(); + assert_eq!( + lineage(&memo, merged), + [ + ColumnLineage::SourceColumn(series_column("latency", 0, "job")), + computed(merged, 1) + ] + ); +} + +/// The PromQL front end lowers `m[5m] @ end()` as `TimeShift { at: End }` +/// above the selector lowered without `@`: it pins when the subtree is +/// read and copies its rows, so it passes the selector's lineage through, +/// as `@ start()` does. +#[test] +fn a_time_shift_at_end_or_start_above_a_selector_passes_its_lineage_through() { + let mut memo = memo(); + let series = memo.find_or_add_group(latency_series()).unwrap(); + let selector = memo.find_or_add_group(range(300_000, series)).unwrap(); + let series_lineage = ["ts", "value", "job"] + .map(|column| ColumnLineage::SourceColumn(series_column("latency", 0, column))); + for at in [AtModifier::End, AtModifier::Start] { + let pinned = memo + .find_or_add_group(shift(0, Some(at), selector)) + .unwrap(); + assert_eq!(lineage(&memo, pinned), series_lineage); + } +} + +/// `SELECT approx_distinct(

_dst) FROM (SELECT a.src AS

_src, a.proto +/// AS

_proto, b.dst AS

_dst, b.proto AS

_other_proto FROM flows a +/// JOIN flows b ON a.src = b.src WHERE .proto = 'tcp')`, aliases +/// prefixed by `prefix`, filtering the left side's `proto` (column 2 of the +/// join) or the right side's (column 5): the HLL state's selection keys. +fn self_join_selection(memo: &mut Memo, prefix: &str, proto_column: usize) -> Vec { + let flows = memo + .find_or_add_group(scan("flows", &["src", "dst", "proto"])) + .unwrap(); + let joined = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(3)), + semantics: ExprSemantics::Sql, + }), + left: flows, + right: flows, + })) + .unwrap(); + let tcp = memo + .find_or_add_group(filter(equals(proto_column, "tcp"), joined)) + .unwrap(); + let renamed = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: [("src", 0), ("proto", 2), ("dst", 4), ("other_proto", 5)] + .map(|(alias, column)| ProjectItem { + alias: Some(format!("{prefix}_{alias}")), + expr: ScalarExpr::Column(column), + }) + .to_vec(), + child: tcp, + })) + .unwrap(); + let state = memo + .find_or_add_group(hll(renamed, vec![], &format!("{prefix}_dst"))) + .unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + selection.columns().keys().cloned().collect() +} + +/// A self-join's two scans of one source have different scan ordinals: the +/// left side's columns are scan 0's, the right side's scan 1's. So a +/// predicate on the left side's `proto` lifts keyed by `flows` scan 0 +/// `proto`, one on the right side's by scan 1, and renaming the columns +/// (other aliases) keeps the key. +#[test] +fn a_self_join_names_its_two_scans_apart() { + let mut memo = memo(); + let flows = memo + .find_or_add_group(scan("flows", &["src", "dst", "proto"])) + .unwrap(); + let joined = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(3)), + semantics: ExprSemantics::Sql, + }), + left: flows, + right: flows, + })) + .unwrap(); + assert_eq!( + lineage(&memo, joined), + [ + (0, "src"), + (0, "dst"), + (0, "proto"), + (1, "src"), + (1, "dst"), + (1, "proto") + ] + .map(|(scan, column)| ColumnLineage::SourceColumn(table_column("flows", scan, column))) + ); + + let left_proto = [table_column("flows", 0, "proto")]; + assert_eq!(self_join_selection(&mut memo, "a", 2), left_proto); + assert_eq!(self_join_selection(&mut memo, "x", 2), left_proto); + assert_eq!( + self_join_selection(&mut memo, "a", 5), + [table_column("flows", 1, "proto")] + ); +} + +/// The memo of the match-grouping examples: `team` declared non-negative +/// in `a` and non-positive in `b`. +fn match_grouping_memo() -> Memo { + let declared = |sign| SourceFacts { + column_signs: BTreeMap::from([("team".to_string(), sign)]), + ..SourceFacts::default() + }; + Memo::new(&DataWorkload { + sources: BTreeMap::from([ + ("a".to_string(), declared(ValueSign::NonNegative)), + ("b".to_string(), declared(ValueSign::NonPositive)), + ]), + ..DataWorkload::default() + }) +} + +/// `Scan metric`: `[ts, value, *labels]`, `ts` its time index. +fn series(metric: &str, labels: &[&str]) -> Operator { + let labels: Vec<(&str, DataType)> = labels + .iter() + .map(|&label| (label, DataType::Utf8)) + .collect(); + memo_fixtures::scan(metric, &labels, vec![]) +} + +/// `lhs * on(job) group_(team) rhs`. +fn grouped_product(side: GroupSide, lhs: MemoGroupId, rhs: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec!["job".into()], + grouping: Some(VectorGrouping { + side, + labels: vec!["team".into()], + }), + }), + }, + return_bool: false, + lhs, + rhs, + }) +} + +/// The lineage and sign of column `name` of `group`. +fn lineage_and_sign_of(memo: &Memo, group: MemoGroupId, name: &str) -> (ColumnLineage, ValueSign) { + let properties = &memo.groups()[group.0 as usize].group_properties; + let column = properties + .schema + .column_id(name) + .expect("a column of that name"); + ( + properties.column_lineage[column].clone(), + properties.column_signs[column], + ) +} + +/// `a * on(job) group_left(team) b`: `team`, which only `b` has, takes +/// `b.team`'s lineage and sign (the "one" side's). +#[test] +fn a_group_left_label_comes_from_the_right_side() { + let mut memo = match_grouping_memo(); + let a = memo.find_or_add_group(series("a", &["job"])).unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Left, a, b)) + .unwrap(); + assert_eq!( + lineage_and_sign_of(&memo, product, "team"), + ( + ColumnLineage::SourceColumn(series_column("b", 0, "team")), + ValueSign::NonPositive + ) + ); +} + +/// `a * on(job) group_left(team) b` with `a` having its own `team`: PromQL +/// overwrites it with `b`'s, so the column takes `b.team`'s lineage and +/// sign in place. +#[test] +fn a_group_left_label_overwrites_the_left_sides_column_of_that_name() { + let mut memo = match_grouping_memo(); + let a = memo + .find_or_add_group(series("a", &["job", "team"])) + .unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Left, a, b)) + .unwrap(); + let schema = &memo.groups()[product.0 as usize].group_properties.schema; + assert_eq!(schema.fields.len(), 4, "no second `team` column"); + assert_eq!( + lineage_and_sign_of(&memo, product, "team"), + ( + ColumnLineage::SourceColumn(series_column("b", 0, "team")), + ValueSign::NonPositive + ) + ); +} + +/// `a * on(job) group_right(team) b`: `team` takes `a.team`'s lineage and +/// sign (the left, "one" side's). +#[test] +fn a_group_right_label_comes_from_the_left_side() { + let mut memo = match_grouping_memo(); + let a = memo + .find_or_add_group(series("a", &["job", "team"])) + .unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Right, a, b)) + .unwrap(); + assert_eq!( + lineage_and_sign_of(&memo, product, "team"), + ( + ColumnLineage::SourceColumn(series_column("a", 0, "team")), + ValueSign::NonNegative + ) + ); +} + +/// `rate(x[])`: per series, `[ts, value, job, team]` kept in place. +fn rate_of_x(memo: &mut Memo, width_ms: i64) -> MemoGroupId { + let x = memo + .find_or_add_group(series("x", &["job", "team"])) + .unwrap(); + let window = memo.find_or_add_group(range(width_ms, x)).unwrap(); + memo.find_or_add_group(aggregate(Reduction::PerEntity, AggIntent::Rate, window)) + .unwrap() +} + +/// `rate(x[5m]) / rate(x[1h])`: both sides read `x`, but a `BinaryOp` +/// does not concatenate their columns; its labels are the left side's, so +/// `job` is still `x` scan 0's. +#[test] +fn a_same_source_binary_op_keeps_its_labels_scan() { + let mut memo = match_grouping_memo(); + let short = rate_of_x(&mut memo, 300_000); + let long = rate_of_x(&mut memo, 3_600_000); + let ratio = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + }, + return_bool: false, + lhs: short, + rhs: long, + })) + .unwrap(); + assert_eq!( + lineage_and_sign_of(&memo, ratio, "job").0, + ColumnLineage::SourceColumn(series_column("x", 0, "job")) + ); +} + +/// `rate(x[5m]) * on(job) group_right(team) rate(x[1h])`: the output's rows +/// are the right ("many") side's, so its labels take that side's lineage, +/// `x` scan 0's, not a shifted scan or a disagreement; `team` comes from +/// the left ("one") side, also `x` scan 0's. +#[test] +fn a_same_source_group_right_takes_the_right_sides_labels() { + let mut memo = match_grouping_memo(); + let short = rate_of_x(&mut memo, 300_000); + let long = rate_of_x(&mut memo, 3_600_000); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Right, short, long)) + .unwrap(); + for label in ["job", "team"] { + assert_eq!( + lineage_and_sign_of(&memo, product, label).0, + ColumnLineage::SourceColumn(series_column("x", 0, label)) + ); + } +} + +/// `a or b` with only `b` having `team`: on `a`'s rows `team` is null, so it +/// is computed by the `or` with no sign, not `b.team` (before, it copied +/// `b.team` and `b`'s sign). +#[test] +fn an_or_label_only_the_right_side_has_is_computed() { + let mut memo = match_grouping_memo(); + let a = memo.find_or_add_group(series("a", &["job"])).unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let union = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Set(PromQLVectorSetOpKind::Or), + vector_match: None, + }, + return_bool: false, + lhs: a, + rhs: b, + })) + .unwrap(); + let team = memo.groups()[union.0 as usize] + .group_properties + .schema + .column_id("team") + .expect("`or` adds the right side's labels"); + assert_eq!( + lineage_and_sign_of(&memo, union, "team"), + (computed(union, team), ValueSign::Unknown) + ); +} diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index ad9e48481..ef8599935 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,7 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod column_lineage; mod memo_aggregate_coverage; mod memo_group_properties; mod physical_export; diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index 642134962..5c96ed40b 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -10,21 +10,21 @@ //! ``` use std::ops::Bound; -use asap_types::ir::operator::arguments::{AtModifier, Reduction}; +use asap_types::ir::operator::arguments::{AtModifier, Reduction, SourceColumn}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::selection::{ColumnIdentity, Constraint, IntervalConstraintLimit}; +use asap_types::ir::properties::selection::{Constraint, IntervalConstraintLimit}; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; use asap_types::stages::{ - view_coverage, AggregateCoverage, CandidateExpression, CandidateForMemoGroup, CoverageError, - LogicalProperties, LogicalRequirements, Memo, MemoGroupError, MemoGroupId, + view_coverage, AggregateCoverage, CandidateExpression, CandidateForMemoGroup, ColumnLineage, + CoverageError, LogicalProperties, LogicalRequirements, Memo, MemoGroupError, MemoGroupId, OriginOfCandidateForMemoGroup, }; use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; -use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; +use asap_types::test_support::memo_fixtures::{filter, range, scan, series_column, shift, state}; use asap_types::workload::DataWorkload; const TS: usize = 0; @@ -145,8 +145,8 @@ fn last_minute() -> Constraint { relative(-MINUTE_MS, 0) } -fn ts() -> ColumnIdentity { - ColumnIdentity::new(None, "ts") +fn ts() -> SourceColumn { + series_column("latency", 0, "ts") } /// The constraint on the timestamp column of `group`'s one selection box. @@ -174,7 +174,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { panic!("one box: {coverage:?}") }; assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], + selection.columns()[&series_column("latency", 0, "job")], jobs(&["api"]) ); assert_eq!(selection.columns()[&ts()], last_minute()); @@ -199,7 +199,7 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { panic!("one box: {coverage:?}") }; assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], + selection.columns()[&series_column("latency", 0, "job")], jobs(&["api", "web"]) ); } @@ -243,7 +243,7 @@ fn a_filter_that_is_no_selection_stays_unlifted() { ] ); assert_eq!( - coverage_of_g10.selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage_of_g10.selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api"]) ); assert!(matches!( @@ -296,19 +296,28 @@ fn states_whose_definitions_read_the_same_subquery_merge() { assert_eq!(definition, coverage(&memo, web).definition()); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, merged).selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api", "web"]) ); } /// PromQL label matchers are the scan's predicates: they are lifted too, so /// `latency{job="api"}` and `latency{job="web"}` states merge. The scan, -/// without them, is the last unlifted operator. +/// without them, is the last unlifted operator. Each scan's `job` column has +/// lineage `latency.job`, so the selections `latency.job ∈ {api}` and +/// `latency.job ∈ {web}` are disjoint selections on one column (concepts doc, +/// section 3, column lineage example 4). #[test] fn scan_predicates_lift_and_their_states_merge() { let mut memo = Memo::new(&DataWorkload::default()); let state_of_job = |memo: &mut Memo, job: &str| { let series = memo.find_or_add_group(latency(vec![job_is(job)])).unwrap(); + assert_eq!( + memo.groups()[series.0 as usize] + .group_properties + .column_lineage[JOB], + ColumnLineage::SourceColumn(series_column("latency", 0, "job")) + ); let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); memo.find_or_add_group(kll(last_minute)).unwrap() }; @@ -321,12 +330,12 @@ fn scan_predicates_lift_and_their_states_merge() { vec![kll_unlifted(), latency(vec![]).map_children(|_| None)] ); assert_eq!( - coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, api).selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api"]) ); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, merged).selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api", "web"]) ); } @@ -490,7 +499,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { } ); assert_eq!( - coverage(&memo, g3).selection()[0].columns()[&ColumnIdentity::new(None, "value")], + coverage(&memo, g3).selection()[0].columns()[&series_column("latency", 0, "value")], Constraint::Interval { lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( 10.0 @@ -571,7 +580,7 @@ fn an_exact_aggregate_has_coverage() { panic!("one box: {coverage:?}") }; assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], + selection.columns()[&series_column("latency", 0, "job")], jobs(&["api"]) ); assert_eq!(selection.columns()[&ts()], last_minute()); diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 005db69a0..3f1808c0f 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -1,7 +1,8 @@ //! `column_signs`, `depends_on_evaluation_time` and `scanned_sources`, //! derived once per memo group from its input groups (concepts doc, section //! 3, "Value sign" and `GroupProperties`; `aggregate_coverage` is in -//! `memo_aggregate_coverage.rs`). Every example reads one of three sources: +//! `memo_aggregate_coverage.rs`; column lineage, derived with the signs, +//! is in `column_lineage.rs`). Every example reads one of three sources: //! //! ```text //! latency [ts, value, host] value declared non-negative @@ -106,7 +107,6 @@ fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: @@ -308,7 +310,7 @@ G10.aggregate_coverage = AggregateCoverage { input: Some(G1), unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], }, - selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], + selection: [ { latency.job ∈ {api}, latency.ts ∈ (t − 60 s, t] } ], } ``` @@ -379,11 +381,11 @@ GROUP BY job; | Condition | Found at | Rule 1: same rows at the `SummaryAgg`? | Rule 2: simple? | Result | |---|---|---|---|---| -| `latency < 100` | `SummaryAgg.filter` | yes, it is already there | yes, an interval | `selection`: `latency ∈ (−∞, 100)` | -| `region = 'us'` | `Scan.predicates` | yes: `Project` passes `region` through (renamed `r`) | yes, a value set | `selection`: `r ∈ {us}` | +| `latency < 100` | `SummaryAgg.filter` | yes, it is already there | yes, an interval | `selection`: `t.latency ∈ (−∞, 100)` | +| `region = 'us'` | `Scan.predicates` | yes: `Project` passes `region` through (renamed `r`) | yes, a value set | `selection`: `t.region ∈ {us}` | | `latency * 2 > 10` | `Scan.predicates` | yes | **no**: it is on an expression, not a column | stays in `definition` | -So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL example with a time window is under **Time** below. +So the `selection` is `t.latency ∈ (−∞, 100)`, `t.region ∈ {us}`. A PromQL example with a time window is under **Time** below. **Rule 1 in detail.** Imagine moving the condition up, one operator at a time, until it is just below the `SummaryAgg`. Every operator it passes must leave the picked rows unchanged. Whether it can pass depends on what the operator does: @@ -391,16 +393,16 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | | `TimeRange` (range) or `TimeShift` (an offset or `@ T`; `@ start()` and `@ end()` are not lifted) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | -| `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | +| `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, on `r`, whose lineage is still `t.region`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | | any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit` | no | | | -A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. How a PromQL comparison is lowered decides whether it is such a `Filter`: +A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. But its condition is on the rate value, a computed column with no source column to name it by, so it stays in `definition` too. How a PromQL comparison is lowered decides whether it is such a `Filter`: | PromQL | Lowered to | Coverage | |---|---|---| -| `rate(m[5m]) > 0` (vector and scalar) | `Filter` | `selection`: `value ∈ (0, ∞)` | +| `rate(m[5m]) > 0` (vector and scalar) | `Filter` | stays in `definition`: `value` is computed by `rate` | | `rate(m[5m]) > bool 0` | `Project` (values become 0 or 1, no row is dropped) | stays in `definition`: it does not pick rows | | `rate(a[5m]) > rate(b[5m])` (vector and vector) | `BinaryOp` (matches the two sides by labels, like a join) | stays in `definition`: the walk does not enter a `BinaryOp` | @@ -426,11 +428,12 @@ Any other shape stays in `definition`: **Which column a condition is on.** -- A column is named by its table and name, `(table, name)`, **as the `SummaryAgg` reads it** (in the schema of its child). Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions. -- A `Project` gives its columns new names and replaces their table with its own qualifier (none by default): after `region AS r`, the column is `(none, r)`, so the condition becomes `r ∈ {us}`. Since the `definition` contains the same `Project`, two states that rename the same way still compare equal. -- Only a `Project` may rename. Any other operator whose output names a column differently from its input (allowed for every operator, e.g. a `Filter` with an explicit schema) stays in `definition`, and so do the conditions at and below it. Removing it would drop the rename: if a `Filter` calls the `tier` column `region`, its `tier = 'eu'` would read as `region ∈ {eu}` and look disjoint from another state's `region ∈ {us}`, though a row with `region = 'us', tier = 'eu'` is in both. -- If two columns the `SummaryAgg` reads have the same `(table, name)` (for example `Project [a AS k, b AS k]`), a condition on `k` cannot tell them apart and stays in `definition`. -- PromQL labels have no table, so a label is named by its name alone. +- A column is named by its **lineage** (concepts doc, section 3, Column lineage) in the group the `SummaryAgg` reads: the source column `(source, scan ordinal, column)` its values are copied from, a `Scan`'s table or metric, which scan of it, and its column there. Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions, though both columns are named `region`. +- The scan ordinal tells a self-join's two scans of one source apart: 0 for the first scan, counted left to right across joins. In `FROM flows a JOIN flows b`, `a.proto` is `(flows, 0, proto)` and `b.proto` is `(flows, 1, proto)`; table aliases do not appear, so renaming them keeps the condition. +- A rename keeps lineage: after `Project [region AS r]`, `r` is still `t.region`, so the condition is `t.region ∈ {us}`. A memo group's schema is derived, so no other operator renames a column. Since the `definition` contains the same `Project`, two states that rename the same way still compare equal. +- A condition on a computed column (a `Project` expression item, an aggregate's measure, a readout's value) has no source column to name and stays in `definition`. Its lineage is `Computed { group, column }`, the memo group and output column that computed it, so two computed columns have equal lineage only when they are the same computation. +- If two columns the `SummaryAgg` reads have the same lineage (for example `Project [a AS k, a AS k2]`), the selection cannot name one of them apart, and their conditions stay in `definition`. +- A PromQL label is named by its metric and label: `latency.job`. - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. @@ -454,11 +457,11 @@ Any other shape stays in `definition`: | Query | Sub-DAG below `SummaryAgg` | `definition` keeps | `selection` takes | |---|---|---|---| -| SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `region ∈ {us}` | -| SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `latency ∈ (−∞, 100)` | -| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `ts ∈ (t − 3m, t − 2m]` | +| SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `t.region ∈ {us}` | +| SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `t.latency ∈ (−∞, 100)` | +| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `m.ts ∈ (t − 3m, t − 2m]` | | PromQL `quantile(0.99, rate(m{job="api"}[5m]))` | `rate(TimeRange(5m, Scan m {job = 'api'}))` | all of it | nothing: `job = 'api'` is under `rate`, which the walk cannot pass yet (Rule 1, later) | -| PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | `rate(TimeRange(5m, Scan m))` | `value ∈ (0, ∞)`: the `Filter` is above `rate` | +| PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | all of it | nothing: the `Filter` is above `rate`, but `value` is computed by `rate`, so no selection can name it | | PromQL `quantile(0.99, rate(a[5m]) > rate(b[5m]))` | `BinaryOp(>, rate(TimeRange(5m, Scan a)), rate(TimeRange(5m, Scan b)))` | all of it | nothing: a vector/vector comparison is a `BinaryOp`, not a `Filter` | `Scan t {…}` is a `Scan` with those predicates. In the last three rows `TimeRange(5m)` stays in `definition` in any case: it is the input window of `rate` and changes the rate values, so it does not just pick rows. @@ -556,7 +559,7 @@ With an error target, the planner answers the quantile from a KLL sketch with `k aggregate coverage of the SummaryAgg ┌──────────────────────────────────────────────────────────────────┐ │ definition: this SummaryAgg over Scan t (the Filter removed) │ -│ selection: region ∈ {us}, latency ∈ (−∞, 10000) │ +│ selection: t.region ∈ {us}, t.latency ∈ (−∞, 10000) │ └──────────────────────────────────────────────────────────────────┘ ``` @@ -795,13 +798,13 @@ The first two build `KLL(latency) by[job]` states. The third can be answered by │ State: job Utf8, state Sketch(KLL k=200) │ [[ SummaryMerge ]] - ▲ selection: region ∈ {us, eu} + ▲ selection: t.region ∈ {us, eu} │ ┌─────────────────┴─────────────────┐ │ │ [[ SummaryAgg ]] [[ SummaryAgg ]] KLL k=200, by job KLL k=200, by job - selection: region ∈ {us} selection: region ∈ {eu} + selection: t.region ∈ {us} selection: t.region ∈ {eu} ▲ ▲ │ │ [ Filter region = 'us' ] [ Filter region = 'eu' ] @@ -1124,9 +1127,6 @@ impl Schema { pub fn is_all_plain(&self) -> bool; /// Position of the first field with this name, or `None`. pub fn column_id(&self, name: &str) -> Option; - /// Position of the field with this table and name, or `None`. Tells apart - /// `a.k` and `b.k` after a join. - pub fn column_id_qualified(&self, table: &str, name: &str) -> Option; } /// One column of a `Schema`. `T` is `FieldDataType` on DAG edges, and plain @@ -1139,13 +1139,9 @@ pub struct Field { pub dtype: T, /// Whether the column may contain NULL. PromQL value columns are never NULL. pub nullable: bool, - /// Table or alias the column comes from (`t` in `t.col`), so two columns - /// with the same name from a join stay apart. `None` for PromQL labels and - /// unqualified columns. - pub table: Option, } impl Field { - /// An unqualified field holding a plain value of type `dtype`. + /// A field holding a plain value of type `dtype`. pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self; /// The value type if the field is plain; `None` for summary state. pub fn plain_dtype(&self) -> Option<&DataType>; @@ -1359,7 +1355,7 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; `SourceColumn` in `crates/types/src/ir/operator/arguments.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/column_properties.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. @@ -1379,15 +1375,28 @@ pub struct SelectionBox { /// unrestricted. /// The timestamp column is one of them; there is no separate time /// field. - pub columns: BTreeMap, + pub columns: BTreeMap, } -/// Names a column across nodes, independent of its position. -pub struct ColumnIdentity { - /// The table or alias the column comes from; `None` if unqualified. - pub table: Option, - /// The column name. - pub name: String, +/// Names a column across nodes, independent of its position and name: the +/// source column of its lineage (`ColumnLineage::SourceColumn`), the same +/// type the lineage holds. +pub struct SourceColumn { + /// The table or metric a `Scan` reads. + pub source: Source, + /// Which scan of `source`: 0 for the first, counted left to right + /// across joins (a self-join's right side is 1). + pub scan_ordinal: u32, + /// The column's name in that source. + pub column: String, +} + +/// Where a memo group's output column's values come from. +pub enum ColumnLineage { + /// Copied unchanged from a column of one scan of a source. + SourceColumn(SourceColumn), + /// Column `column` of memo group `group`, where it was computed. + Computed { group: MemoGroupId, column: ColumnId }, } /// A constraint on one column. diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index cb822d7a6..52db97989 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -105,7 +105,7 @@ enum NonASAPOp { Values { rows: Vec>, schema: Schema }, Filter { child: Rc, pred: Predicate }, Project { - child: Rc, cols: Vec, qualifier: Option, + child: Rc, cols: Vec, }, Aggregate { child: Rc, reduction: Reduction, measures: Vec, @@ -145,8 +145,9 @@ Ordinary payload fields have these roles: `measures` describes the requested aggregates. Grouping is distinct from ordering or limiting within groups, represented by `partition_by`. - Join/set kinds, vector matching, window frames and time selections preserve - source-language semantics. Output names, qualifiers and proven uniqueness also - survive optimization. + source-language semantics. Output names and proven uniqueness also survive + optimization; a column's source is its lineage (a memo-group property), not a + table qualifier on the schema. `ASAPOp` describes state construction, state operations and evaluation separately. `FieldDataType` (§2.1) types every output field; state-producing operations use its diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index f3346279b..1eecb5935 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -75,7 +75,6 @@ pub struct Field { pub name: String, pub dtype: FieldDataType, // Plain(DataType) | ExactAggregate(..) | Sketch(..) | SummarySample(..) | Wavelet(..) | StatModel(..) pub nullable: bool, - pub table: Option, // SQL table/alias qualifier; None for PromQL labels } ``` @@ -156,9 +155,10 @@ gated by `NonASAPOp::is_shareable_when_equal` for non-ASAP producers. ## Fields and column references -`Schema` holds `Field` metadata (name, type, nullability, qualifier) and no -values; an unresolved `ColumnRef` resolves to a positional `ColumnId` within one -schema. The design, including how the same position selects a runtime value, is +`Schema` holds `Field` metadata (name, type, nullability) and no values; an +unresolved `ColumnRef` resolves to a positional `ColumnId` within one schema. A +front end binds a qualified `table.column` with its own `Scope` while it +lowers; the IR keeps no qualifier. The design, including how the same position selects a runtime value, is in [Schema and physical data for ASAP primitives](../design_docs/proposals/asap-primitive-schema.md#3-proposed-schema-design). ## Node index @@ -567,7 +567,6 @@ SELECT srcip, dstip FROM packets **Fields:** - `cols` — the output column list (expression + optional alias per column). -- `qualifier` — a table alias re-qualifying every output column, for a derived table; `None` for an ordinary `SELECT` list. - `child` — the input being projected. ### BinaryOp