From 03163fd1ee231ba30a46f5b3d1d1f43d65b44c7a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:34:48 +0000 Subject: [PATCH 1/5] refactor(types, frontends): column lineage replaces table qualifiers (#1207 PR 2b) GroupProperties gains column_lineage (one ColumnLineage per output column, derived once from the input groups'). Lineage and column_signs are derived together in stages/logical/column_properties.rs by main's one per-operator column layout (aggregate_output / summary_agg_layout and OutputColumnSource): a copied column keeps its lineage and sign, a computed one is Computed with the operator's sign rule. Chains compute window_end and their state and keep their keys' lineage; a TimeRange or TimeShift over a chain, and TimeShift { at: End | Start } above a selector, pass lineage through; a merge of a chain's selected windows has the chain build's layout; a label of `or` or group_right keeps what both sides agree on. SourceColumn { source: Source, column } is in ir/operator/arguments.rs beside Source. ColumnLineage is a memo-scope property beside ValueSign in stages/logical/column_properties.rs; the memo's coverage derivation reads each input group's stored lineage through MemoGroupAsInput, with no OperatorInput accessor, and carries it up through the liftable operators above the aggregate's input (lineage_at_top). Aggregate coverage names a selection's column by its SourceColumn lineage; ColumnIdentity is deleted. A predicate on a Computed column, or on one of two columns of one lineage, is not lifted; the regrouping key check compares lineage. Schema fields and the IR Project lose their table qualifier; the front ends bind qualified names with their own scope while lowering. Declared signs stay keyed by source name and column (SourceFacts). --- crates/devtools/src/planner_fuzz/relations.rs | 8 +- crates/devtools/tests/viewer_contract.rs | 1 - crates/executor/src/expressions/columnar.rs | 1 - .../src/operators/aggregate/temporal.rs | 2 - crates/executor/src/operators/common.rs | 1 - crates/executor/src/operators/group_table.rs | 2 - .../executor/src/operators/series_labels.rs | 1 - crates/executor/src/operators/summary/mod.rs | 2 - crates/executor/src/values/batch.rs | 8 - .../executor/tests/aggregate_memory_limit.rs | 1 - crates/executor/tests/blocking_resources.rs | 3 - .../tests/cheap_measures_memory_limit.rs | 1 - crates/executor/tests/current_series_heap.rs | 1 - crates/executor/tests/deployment.rs | 1 - crates/executor/tests/exact_state_columns.rs | 1 - .../tests/grouped_limit_memory_limit.rs | 1 - crates/executor/tests/hydra_cms_execution.rs | 1 - .../tests/keyed_summary_row_weight.rs | 1 - crates/executor/tests/physical_dag.rs | 3 - crates/executor/tests/physical_semantics.rs | 2 - .../executor/tests/precompute_population.rs | 1 - crates/executor/tests/promql_binary.rs | 4 - crates/executor/tests/raw_scan.rs | 2 - .../tests/series_window_memory_limit.rs | 1 - crates/executor/tests/state_column_updater.rs | 1 - .../stored_tumbling_window_memory_limit.rs | 2 - .../tests/summary_build_memory_limit.rs | 1 - crates/executor/tests/summary_projection.rs | 4 - crates/executor/tests/text_extremum_memory.rs | 1 - crates/executor/tests/univmon_execution.rs | 1 - crates/frontend-common/src/canonicalize.rs | 7 - crates/frontend-common/src/resolve.rs | 574 ++++++++++++------ crates/frontend-common/src/schema_resolver.rs | 1 + crates/frontend-common/src/unresolved.rs | 7 + crates/frontend-metricsql/src/lib.rs | 1 + crates/frontend-promql/src/promql.rs | 1 + crates/frontend-sql/src/frequency.rs | 3 +- crates/frontend-sql/src/sql/mod.rs | 23 +- crates/frontend-sql/tests/sql_lowering.rs | 30 +- crates/planner/src/lib.rs | 3 +- crates/types/src/accuracy/target.rs | 1 - crates/types/src/ir/flat.rs | 1 - crates/types/src/ir/operator/agg_intent.rs | 8 +- crates/types/src/ir/operator/arguments.rs | 18 +- crates/types/src/ir/operator/node.rs | 25 +- crates/types/src/ir/operator/non_asap.rs | 39 +- crates/types/src/ir/properties/selection.rs | 45 +- .../types/src/ir/scalar/column_resolution.rs | 33 +- crates/types/src/ir/scalar/expr_ir.rs | 7 +- crates/types/src/ir/scalar/mod.rs | 1 - crates/types/src/ir/schema/mod.rs | 54 +- crates/types/src/ir/schema/topk.rs | 10 + .../src/stages/logical/aggregate_coverage.rs | 129 ++-- .../src/stages/logical/column_properties.rs | 390 ++++++++---- crates/types/src/stages/logical/group.rs | 12 +- crates/types/src/stages/logical/memo/tests.rs | 2 - crates/types/src/stages/logical/mod.rs | 2 +- crates/types/src/stages/mod.rs | 2 +- crates/types/src/test_support.rs | 8 +- crates/types/tests/column_lineage.rs | 419 +++++++++++++ crates/types/tests/main.rs | 1 + crates/types/tests/memo_aggregate_coverage.rs | 55 +- crates/types/tests/memo_group_properties.rs | 1 - crates/types/tests/physical_export.rs | 1 - crates/types/tests/schema_rebuilding.rs | 2 - crates/types/tests/structure_contract.rs | 2 - .../proposals/asap-primitive-schema.md | 81 ++- 67 files changed, 1373 insertions(+), 686 deletions(-) create mode 100644 crates/types/tests/column_lineage.rs diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index 676b5805e..cdd8e3782 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -954,12 +954,7 @@ fn with_root_estimate( estimate: Rc, dropped: &[String], ) -> Result, String> { - let Some(NonASAPOp::Project { - cols, - qualifier, - child, - }) = root.non_asap() - else { + let Some(NonASAPOp::Project { cols, child }) = root.non_asap() else { return Ok(estimate); }; let mut kept = Vec::new(); @@ -980,7 +975,6 @@ fn with_root_estimate( let cols = kept; OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { cols, - qualifier: qualifier.clone(), child: estimate, })) .map_err(|e| format!("projection over the new estimate: {e}")) diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index 4d3862e64..440e4db60 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -42,7 +42,6 @@ fn non_asap_samples() -> Vec> { }, NonASAPOp::Project { cols: vec![], - qualifier: None, child: (), }, NonASAPOp::Aggregate { diff --git a/crates/executor/src/expressions/columnar.rs b/crates/executor/src/expressions/columnar.rs index dbb37b317..d718588d2 100644 --- a/crates/executor/src/expressions/columnar.rs +++ b/crates/executor/src/expressions/columnar.rs @@ -813,7 +813,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/aggregate/temporal.rs b/crates/executor/src/operators/aggregate/temporal.rs index b913fad41..12771677e 100644 --- a/crates/executor/src/operators/aggregate/temporal.rs +++ b/crates/executor/src/operators/aggregate/temporal.rs @@ -341,13 +341,11 @@ mod tests { name: "time".into(), dtype: SummaryFamilyType::Plain(DataType::Timestamp), nullable: false, - table: None, }, SummaryField { name: "value".into(), dtype: SummaryFamilyType::Plain(DataType::Float64), nullable: false, - table: None, }, ], time_index: Some(0), diff --git a/crates/executor/src/operators/common.rs b/crates/executor/src/operators/common.rs index f0182757d..0a9732dc5 100644 --- a/crates/executor/src/operators/common.rs +++ b/crates/executor/src/operators/common.rs @@ -15,7 +15,6 @@ pub(super) fn result_field(name: &str, dtype: DataType, nullable: bool) -> Summa name: name.into(), dtype: SummaryFamilyType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/group_table.rs b/crates/executor/src/operators/group_table.rs index 64a173c97..53e6a702d 100644 --- a/crates/executor/src/operators/group_table.rs +++ b/crates/executor/src/operators/group_table.rs @@ -476,7 +476,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: true, - table: None, }; let schema = Arc::new(Schema { fields: vec![ @@ -548,7 +547,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let schema = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/src/operators/series_labels.rs b/crates/executor/src/operators/series_labels.rs index f44a87933..fdf8b8b73 100644 --- a/crates/executor/src/operators/series_labels.rs +++ b/crates/executor/src/operators/series_labels.rs @@ -1130,7 +1130,6 @@ mod columnar_tests { name: name.into(), dtype: SummaryFamilyType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 66ae98b6b..d7b94f9ac 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -92,7 +92,6 @@ impl Operator { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }); Ok(Self { kind: Kind::AllGroupsSummaryBuild { @@ -176,7 +175,6 @@ impl Operator { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }); Ok(Self { kind: Kind::PerGroupSummaryBuild { diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index 77aa4e5fc..25d221e61 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -617,7 +617,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable, - table: None, } } @@ -663,13 +662,11 @@ mod tests { name: "a".into(), dtype: DataType::Int64, nullable: false, - table: None, }, Field:: { name: "b".into(), dtype: DataType::Utf8, nullable: true, - table: None, }, ]; let labels = DataType::Map { @@ -787,7 +784,6 @@ mod tests { name: "state".into(), dtype: dd_family(), nullable: false, - table: None, }, ], time_index: None, @@ -869,7 +865,6 @@ mod tests { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }, ], time_index: None, @@ -1011,7 +1006,6 @@ mod tests { name: "state".into(), dtype: family, nullable: false, - table: None, }, ], time_index: None, @@ -1064,7 +1058,6 @@ mod tests { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }], time_index: None, unique_keys: vec![], @@ -1143,7 +1136,6 @@ mod tests { name: "state".into(), dtype: family, nullable: false, - table: None, }], time_index: None, unique_keys: vec![], diff --git a/crates/executor/tests/aggregate_memory_limit.rs b/crates/executor/tests/aggregate_memory_limit.rs index 75e6268b9..ac4a75085 100644 --- a/crates/executor/tests/aggregate_memory_limit.rs +++ b/crates/executor/tests/aggregate_memory_limit.rs @@ -53,7 +53,6 @@ fn run_aggregate( name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index 7926c0416..e040021f9 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -20,7 +20,6 @@ fn schema(width: usize) -> SchemaRef { closed: false, fields: (0..width) .map(|i| Field { - table: None, name: format!("v{i}"), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, @@ -193,13 +192,11 @@ fn weighted_summary_build_yields_within_a_batch() { closed: false, fields: vec![ Field { - table: None, name: "item".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, }, Field { - table: None, name: "weight".into(), dtype: FieldDataType::Plain(DataType::Float64), nullable: false, diff --git a/crates/executor/tests/cheap_measures_memory_limit.rs b/crates/executor/tests/cheap_measures_memory_limit.rs index 070950a98..873286fef 100644 --- a/crates/executor/tests/cheap_measures_memory_limit.rs +++ b/crates/executor/tests/cheap_measures_memory_limit.rs @@ -90,7 +90,6 @@ fn many_groups_of_cheap_measures_fail_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![field("k", DataType::Int64), field("v", DataType::Float64)], diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 2b18c3592..87d41f5b6 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -29,7 +29,6 @@ fn schema() -> Arc { ] .into_iter() .map(|(name, dtype)| Field { - table: None, name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index 11fd6e189..f158126be 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -247,7 +247,6 @@ fn heap_sketch_states_load_as_dag_summary_values() { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }], unique_keys: vec![], closed: false, diff --git a/crates/executor/tests/exact_state_columns.rs b/crates/executor/tests/exact_state_columns.rs index 25e89e23d..ce1bcfa61 100644 --- a/crates/executor/tests/exact_state_columns.rs +++ b/crates/executor/tests/exact_state_columns.rs @@ -16,7 +16,6 @@ fn field(name: &str, dtype: DataType) -> Field { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, } } diff --git a/crates/executor/tests/grouped_limit_memory_limit.rs b/crates/executor/tests/grouped_limit_memory_limit.rs index 10f5ceecf..e1b03188a 100644 --- a/crates/executor/tests/grouped_limit_memory_limit.rs +++ b/crates/executor/tests/grouped_limit_memory_limit.rs @@ -43,7 +43,6 @@ fn many_groups_in_one_batch_fail_near_the_memory_limit() { name: "k".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, - table: None, }], time_index: None, unique_keys: vec![], diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index b7ca69528..0696d2816 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -117,7 +117,6 @@ fn summary_agg(update: SummaryUpdate, filter: Option>) -> Physi name: "state".into(), dtype: family(), nullable: false, - table: None, }, ]), ) diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 0733da866..e730108e4 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -157,7 +157,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { name: "state".into(), dtype: heap_family(), nullable: false, - table: None, }, ])) .clone(), diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 517067f4e..b77a7587b 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -26,7 +26,6 @@ fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { fields: fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: FieldDataType::Plain(dtype.clone()), nullable: *nullable, @@ -466,7 +465,6 @@ fn exact_state_and_family_validation() { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "state".into(), dtype: family.clone(), nullable: false, @@ -564,7 +562,6 @@ fn bind_physical_asap_dag_before_execution() { )), }, }], - qualifier: None, child: 0, }), ), diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index a9b0ccc02..1413688fb 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -25,7 +25,6 @@ fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { fields: fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: FieldDataType::Plain(dtype.clone()), nullable: *nullable, @@ -926,7 +925,6 @@ fn planner_node( fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: dtype.clone(), nullable: *nullable, diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index e9af0acad..938cf0b5a 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -23,7 +23,6 @@ fn logical_schema(family: FieldDataType) -> Schema { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "value".into(), dtype: family, nullable: false, diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 29c23131b..d2c03d033 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -25,7 +25,6 @@ fn schema() -> SchemaRef { closed: false, fields: vec![ Field { - table: None, name: "labels".into(), dtype: FieldDataType::Plain(DataType::Map { key: Box::new(DataType::Utf8), @@ -35,7 +34,6 @@ fn schema() -> SchemaRef { nullable: false, }, Field { - table: None, name: "value".into(), dtype: FieldDataType::Plain(DataType::Float64), nullable: false, @@ -346,13 +344,11 @@ fn stored_series_evaluations_support_filters_and_sets() { closed: false, fields: vec![ Field { - table: None, name: PROMQL_SERIES_IDENTITY.into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, }, Field { - table: None, name: "value".into(), dtype: family.clone(), nullable: false, diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 830b799be..10bc11ffe 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -35,7 +35,6 @@ fn fixture() -> (planner_types::ir::NonASAPOp, SchemaRef, Vec) { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "value".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: true, @@ -829,7 +828,6 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { dag.edges.retain(|edge| edge.consumer < 3); let family = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let state = planner_types::ir::schema::Schema::new(vec![Field { - table: None, name: "count".into(), dtype: family.clone(), nullable: false, diff --git a/crates/executor/tests/series_window_memory_limit.rs b/crates/executor/tests/series_window_memory_limit.rs index e8c83bcd5..07ca7d29f 100644 --- a/crates/executor/tests/series_window_memory_limit.rs +++ b/crates/executor/tests/series_window_memory_limit.rs @@ -40,7 +40,6 @@ fn a_long_subquery_over_many_series_fails_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index 6569fcecc..076b8633d 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -27,7 +27,6 @@ fn plain(name: &str, dtype: DataType) -> Field { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, } } diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index f5391cf28..4c385fca6 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -33,13 +33,11 @@ fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { name: "series".into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, - table: None, }, Field { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }, ], time_index: None, diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index 146e067ba..e08176203 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -41,7 +41,6 @@ fn one_large_batch_fails_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 1ab25998e..c624a6d9e 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -85,13 +85,11 @@ fn summary_projection_survives_recovery() { closed: false, fields: vec![ Field { - table: None, name: "state".into(), dtype: family.clone(), nullable: false, }, Field { - table: None, name: "service".into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, @@ -152,7 +150,6 @@ fn summary_projection_survives_recovery() { expr: planner_types::ir::ScalarExpr::Column(index), }) .collect(), - qualifier: None, child: 0, }), output_schema: output.clone(), @@ -251,7 +248,6 @@ fn summary_merge_without_inputs_is_rejected() { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "state".into(), dtype: family, nullable: false, diff --git a/crates/executor/tests/text_extremum_memory.rs b/crates/executor/tests/text_extremum_memory.rs index 7aec609c3..0cc056fa4 100644 --- a/crates/executor/tests/text_extremum_memory.rs +++ b/crates/executor/tests/text_extremum_memory.rs @@ -65,7 +65,6 @@ fn text_max_charges_only_the_current_value() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![field("k", DataType::Int64), field("t", DataType::Utf8)], diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 341977372..49241d6a5 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -158,7 +158,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() false, )])); let state = Arc::new(Schema::new(vec![Field { - table: None, name: "state".into(), dtype: family(), nullable: false, diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 6123568cb..70bcf40f6 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -314,7 +314,6 @@ fn try_rewrite_row_number_top_k( ) -> Result>, SchemaDerivationError> { let Some(NonASAPOp::Project { cols: select_list, - qualifier, child: filter, }) = node.non_asap() else { @@ -427,14 +426,12 @@ fn try_rewrite_row_number_top_k( (Some(partition), Some(order)) if !partition_by.is_without() => { let selected = new(NonASAPOp::Project { cols: kept, - qualifier: qualifier.clone(), child: Rc::clone(ranked), })?; limit(GroupKeys::by(partition), order, selected)? } _ => new(NonASAPOp::Project { cols: kept, - qualifier: qualifier.clone(), child: limit(partition_by.clone(), order_by.clone(), Rc::clone(ranked))?, })?, }; @@ -636,7 +633,6 @@ mod tests { expr: ScalarExpr::Column(1), }, ], - qualifier: None, child, }) } @@ -754,7 +750,6 @@ mod tests { alias: None, expr: ScalarExpr::Column(0), }], - qualifier: None, child: Rc::clone(&ranking), }); let out = canonicalize(Rc::clone(&q)).unwrap(); @@ -982,7 +977,6 @@ mod tests { expr: ScalarExpr::Column(col), }) .collect(), - qualifier: None, child, }) } @@ -1220,7 +1214,6 @@ mod tests { let sub = one_column_subquery(); let root = node(NonASAPOp::Project { child: scan(), - qualifier: None, cols: vec![ ProjectItem { alias: Some("a".into()), diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 79010baa4..76b6a7f88 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -20,8 +20,8 @@ use std::rc::Rc; use thiserror::Error; -use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction}; -use asap_types::ir::scalar::{resolve_column_ref, resolve_column_refs, ColumnRef, ResolveError}; +use asap_types::ir::operator::{AggIntent, GroupKeys, JoinKind, Reduction}; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ResolveError}; use asap_types::ir::schema::{ColumnId, Schema}; use asap_types::ir::SchemaDerivationError; use asap_types::ir::{NonASAPOp, OperatorNode, Predicate, ProjectItem, ScalarExpr, SortKey}; @@ -43,23 +43,102 @@ pub enum ResolveDAGError { use crate::canonicalize::canonicalize; +/// The names an operator's parent resolves against: the operator's output +/// schema and, per column, the SQL table or alias that qualifies it (`None` +/// when unqualified; an empty list qualifies no column). A qualified +/// reference (`f.src`) binds here, while lowering; the IR keeps no qualifier. +#[derive(Clone, Copy)] +struct Scope<'a> { + schema: &'a Schema, + qualifiers: &'a [Option], +} + +impl<'a> Scope<'a> { + /// A scope whose columns have no qualifier. + fn unqualified(schema: &'a Schema) -> Self { + Self { + schema, + qualifiers: &[], + } + } + + /// The column `column` names: by qualifier and name when it is + /// qualified and a column has both, else by its unique name. + fn resolve(&self, column: &ColumnRef) -> Result { + if let ColumnRef::Qualified { table, name } = column { + let qualified = + self.schema + .fields + .iter() + .zip(self.qualifiers) + .position(|(field, qualifier)| { + field.name == *name && qualifier.as_deref() == Some(table.as_str()) + }); + if let Some(id) = qualified { + return Ok(id); + } + } + resolve_column_ref(column, self.schema) + } + + fn resolve_all(&self, columns: &[ColumnRef]) -> Result, ResolveError> { + columns.iter().map(|column| self.resolve(column)).collect() + } + + /// The qualifier of column `column`. + fn qualifier(&self, column: ColumnId) -> Option { + self.qualifiers.get(column).cloned().flatten() + } +} + +/// A resolved operator with the qualifier of each output column: the scope +/// its parent resolves names in. +struct Bound { + node: Rc, + qualifiers: Vec>, +} + +impl Bound { + /// `node` with `qualifiers`; unqualified unless there is one per column. + fn new(node: Rc, qualifiers: Vec>) -> Self { + let qualifiers = if qualifiers.len() == node.schema.fields.len() { + qualifiers + } else { + Vec::new() + }; + Self { node, qualifiers } + } + + fn unqualified(node: Rc) -> Self { + Self::new(node, Vec::new()) + } + + fn scope(&self) -> Scope<'_> { + Scope { + schema: &self.node.schema, + qualifiers: &self.qualifiers, + } + } +} + /// Resolve the whole tree rooted at `tree`: bind every `ColumnRef` to a /// `ColumnId` via the usage-derived schema resolver, then canonicalize the result. pub fn resolve_root(tree: &UnresolvedOp) -> Result, ResolveDAGError> { - resolve_root_with_inherited(tree, &[]) + Ok(resolve_root_with_inherited(tree, &[])?.node) } /// [`resolve_root`] with label names inherited from an enclosing scope seeded /// into the leaf schema (a `BinaryOp` side, a scalar operand's operator). +/// Canonicalizing keeps the root's output columns, so their qualifiers. fn resolve_root_with_inherited( tree: &UnresolvedOp, inherited: &[String], -) -> Result, ResolveDAGError> { +) -> Result { let usage_derived_schema = resolve_schema_with_inherited(tree, inherited); let root = resolve(tree, &usage_derived_schema)?; - let root = canonicalize(root)?; - root.validate_structure()?; - Ok(root) + let node = canonicalize(root.node)?; + node.validate_structure()?; + Ok(Bound::new(node, root.qualifiers)) } /// Bind `tree` as a root in its own scope, inheriting from `enclosing` the @@ -73,7 +152,7 @@ fn resolve_nested_root( .into_iter() .filter(|n| !own.contains(n)) .collect(); - resolve_root_with_inherited(tree, &inherited) + Ok(resolve_root_with_inherited(tree, &inherited)?.node) } fn node(op: NonASAPOp) -> Result, ResolveDAGError> { @@ -83,31 +162,32 @@ fn node(op: NonASAPOp) -> Result, ResolveDAGError> { } /// The generic substitution walk. `usage_derived_schema` is the schema a -/// schemaless `Scan` in this scope binds to. -fn resolve( - tree: &UnresolvedOp, - usage_derived_schema: &Schema, -) -> Result, ResolveDAGError> { +/// schemaless `Scan` in this scope binds to. Qualifiers follow the columns: +/// a `Scan` qualifies its columns by its own; an operator that passes +/// columns through, a join and an aggregate's group keys keep them; a +/// `Project` qualifies all its columns by its own or none. +fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result { use UnresolvedOp as U; - let expr = - |e: &UnresolvedScalar, schema: &Schema| resolve_expr_in(e, schema, usage_derived_schema); - let pred = |p: &UnresolvedScalar, schema: &Schema| { - Ok::<_, ResolveDAGError>(Predicate(expr(p, schema)?)) - }; - let sort_keys = |keys: &[UnresolvedSortKey], schema: &Schema| { + let expr = |e: &UnresolvedScalar, scope: Scope| resolve_expr_in(e, scope, usage_derived_schema); + let pred = + |p: &UnresolvedScalar, scope: Scope| Ok::<_, ResolveDAGError>(Predicate(expr(p, scope)?)); + let sort_keys = |keys: &[UnresolvedSortKey], scope: Scope| { keys.iter() .map(|k| { Ok::<_, ResolveDAGError>(SortKey { - expr: expr(&k.expr, schema)?, + expr: expr(&k.expr, scope)?, ascending: k.ascending, nulls_first: k.nulls_first, }) }) .collect::, _>>() }; + // A node over `child` with `child`'s columns and qualifiers. + let passing = |op: NonASAPOp, child: Bound| Ok(Bound::new(node(op)?, child.qualifiers)); + let unqualified = |op: NonASAPOp| Ok(Bound::unqualified(node(op)?)); // A PromQL child over an open schema gets an opaque series-identity column. let promql_series = |child: &UnresolvedOp| -> Result, ResolveDAGError> { - let child = resolve(child, usage_derived_schema)?; + let child = resolve(child, usage_derived_schema)?.node; if child.schema.closed { return Ok(child); } @@ -119,19 +199,28 @@ fn resolve( source, predicates, schema, + qualifier, } => { let schema = schema .clone() .unwrap_or_else(|| usage_derived_schema.clone()); + let qualifiers = vec![qualifier.clone(); schema.fields.len()]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let predicates = predicates .iter() - .map(|p| pred(&p.0, &schema)) + .map(|p| pred(&p.0, scope)) .collect::, _>>()?; - node(NonASAPOp::Scan { - source: source.clone(), - predicates, - schema, - }) + Ok(Bound::new( + node(NonASAPOp::Scan { + source: source.clone(), + predicates, + schema, + })?, + qualifiers, + )) } // Row expressions have no input-column scope. @@ -141,11 +230,11 @@ fn resolve( .iter() .map(|row| { row.iter() - .map(|e| expr(e, &empty)) + .map(|e| expr(e, Scope::unqualified(&empty))) .collect::, _>>() }) .collect::, _>>()?; - node(NonASAPOp::Values { + unqualified(NonASAPOp::Values { rows, schema: schema.clone(), }) @@ -162,7 +251,13 @@ fn resolve( } => { let child = promql_series(child)?; let scalar = resolve_expr(scalar, &Schema::default())?; - lower_scalar_vector(child, scalar, op, *scalar_left, *return_bool) + Ok(Bound::unqualified(lower_scalar_vector( + child, + scalar, + op, + *scalar_left, + *return_bool, + )?)) } U::PromqlMap { child, @@ -171,42 +266,61 @@ fn resolve( } => { let child = promql_series(child)?; let sample = resolve_expr(sample, &child.schema)?; - project_sample(child, sample, *drop_metric_name) + Ok(Bound::unqualified(project_sample( + child, + sample, + *drop_metric_name, + )?)) } - U::PromqlVectorFromScalar(inner) => node(NonASAPOp::PromqlVectorFromScalar(expr( + U::PromqlVectorFromScalar(inner) => unqualified(NonASAPOp::PromqlVectorFromScalar(expr( inner, - usage_derived_schema, + Scope::unqualified(usage_derived_schema), )?)), U::PromqlRelabel { dst, value, child } => { let child = resolve(child, usage_derived_schema)?; - let value = expr(value, &child.schema)?; - node(NonASAPOp::PromqlRelabel { + let value = expr(value, child.scope())?; + unqualified(NonASAPOp::PromqlRelabel { dst: dst.clone(), value, - child, + child: child.node, }) } - U::PromqlInfoEnrich { selector, child } => node(NonASAPOp::PromqlInfoEnrich { - selector: selector.clone(), - child: resolve(child, usage_derived_schema)?, - }), + U::PromqlInfoEnrich { selector, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::PromqlInfoEnrich { + selector: selector.clone(), + child: Rc::clone(&child.node), + }, + child, + ) + } U::PromqlSeriesSample { by, kind, child } => { let child = resolve(child, usage_derived_schema)?; - let by = resolve_group_keys(by, &child.schema)?; - node(NonASAPOp::PromqlSeriesSample { - by, - kind: *kind, + let by = resolve_group_keys(by, child.scope())?; + passing( + NonASAPOp::PromqlSeriesSample { + by, + kind: *kind, + child: Rc::clone(&child.node), + }, child, - }) + ) } U::Filter { pred: p, child } => { let child = resolve(child, usage_derived_schema)?; - let pred = pred(&p.0, &child.schema)?; - node(NonASAPOp::Filter { pred, child }) + let pred = pred(&p.0, child.scope())?; + passing( + NonASAPOp::Filter { + pred, + child: Rc::clone(&child.node), + }, + child, + ) } U::Project { @@ -220,15 +334,18 @@ fn resolve( .map(|item| { Ok::<_, ResolveDAGError>(ProjectItem { alias: item.alias.clone(), - expr: expr(&item.expr, &child.schema)?, + expr: expr(&item.expr, child.scope())?, }) }) .collect::, _>>()?; - node(NonASAPOp::Project { - cols, - qualifier: qualifier.clone(), - child, - }) + let qualifiers = vec![qualifier.clone(); cols.len()]; + Ok(Bound::new( + node(NonASAPOp::Project { + cols, + child: child.node, + })?, + qualifiers, + )) } U::Aggregate { @@ -240,42 +357,76 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let reduction = resolve_reduction(reduction, &child.schema)?; + let scope = child.scope(); + let reduction = resolve_reduction(reduction, scope)?; let measures = measures .iter() - .map(|m| resolve_agg_intent(m, &child.schema)) + .map(|m| resolve_agg_intent(m, scope)) .collect::, ResolveError>>()?; let filters = filters .iter() - .map(|p| p.as_ref().map(|p| pred(&p.0, &child.schema)).transpose()) + .map(|p| p.as_ref().map(|p| pred(&p.0, scope)).transpose()) .collect::, _>>()?; + // A `by` key keeps its qualifier; a per-series reduction keeps + // its input's columns in place. A top-k emits ranked rows. + let qualifiers: Vec> = match &reduction { + _ if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) => Vec::new(), + Reduction::PerEntity => child.qualifiers.clone(), + Reduction::Reduce(by) if by.is_without() => Vec::new(), + Reduction::Reduce(by) => by + .keys() + .iter() + .map(|&key| scope.qualifier(key)) + .chain(std::iter::repeat_n(None, measures.len())) + .collect(), + }; // HAVING is evaluated over the aggregate's own output. let having = having .as_ref() .map(|h| { let out_schema = aggregate_output_schema( - &child.schema, + &child.node.schema, &reduction, &measures, output_names, )?; - pred(&h.0, &out_schema) + let out_qualifiers = if qualifiers.len() == out_schema.fields.len() { + &qualifiers[..] + } else { + &[] + }; + pred( + &h.0, + Scope { + schema: &out_schema, + qualifiers: out_qualifiers, + }, + ) }) .transpose()?; - node(NonASAPOp::Aggregate { - reduction, - measures, - output_names: output_names.clone(), - filters, - having, - child, - }) + Ok(Bound::new( + node(NonASAPOp::Aggregate { + reduction, + measures, + output_names: output_names.clone(), + filters, + having, + child: child.node, + })?, + qualifiers, + )) } U::Dedup { cols, child } => { let child = resolve(child, usage_derived_schema)?; - let cols = resolve_column_refs(cols, &child.schema)?; - node(NonASAPOp::Dedup { cols, child }) + let cols = child.scope().resolve_all(cols)?; + passing( + NonASAPOp::Dedup { + cols, + child: Rc::clone(&child.node), + }, + child, + ) } U::Concat { children } => { @@ -283,7 +434,15 @@ fn resolve( .iter() .map(|c| resolve(c, usage_derived_schema)) .collect::, _>>()?; - node(NonASAPOp::Concat { children }) + let qualifiers = children + .first() + .map_or_else(Vec::new, |first| first.qualifiers.clone()); + Ok(Bound::new( + node(NonASAPOp::Concat { + children: children.into_iter().map(|child| child.node).collect(), + })?, + qualifiers, + )) } U::Join { @@ -296,15 +455,37 @@ fn resolve( // sets); the predicate sees left ++ right. let left = resolve_root_with_inherited(left, &[])?; let right = resolve_root_with_inherited(right, &[])?; - let mut concat = left.schema.clone(); - concat.fields.extend(right.schema.fields.iter().cloned()); - let pred = pred(&p.0, &concat)?; - node(NonASAPOp::Join { - kind: kind.clone(), - pred, - left, - right, - }) + let qualify = |side: &Bound| { + (0..side.node.schema.fields.len()) + .map(|column| side.scope().qualifier(column)) + .collect::>() + }; + let mut concat = left.node.schema.clone(); + concat + .fields + .extend(right.node.schema.fields.iter().cloned()); + let mut qualifiers = qualify(&left); + qualifiers.extend(qualify(&right)); + let pred = pred( + &p.0, + Scope { + schema: &concat, + qualifiers: &qualifiers, + }, + )?; + // A semi or anti join outputs the left side's columns. + if matches!(kind, JoinKind::Semi | JoinKind::Anti) { + qualifiers.truncate(left.node.schema.fields.len()); + } + Ok(Bound::new( + node(NonASAPOp::Join { + kind: kind.clone(), + pred, + left: left.node, + right: right.node, + })?, + qualifiers, + )) } U::SetOp { @@ -312,12 +493,19 @@ fn resolve( all, left, right, - } => node(NonASAPOp::SetOp { - kind: kind.clone(), - all: *all, - left: resolve_root_with_inherited(left, &[])?, - right: resolve_root_with_inherited(right, &[])?, - }), + } => { + let left = resolve_root_with_inherited(left, &[])?; + let right = resolve_root_with_inherited(right, &[])?; + Ok(Bound::new( + node(NonASAPOp::SetOp { + kind: kind.clone(), + all: *all, + left: left.node, + right: right.node, + })?, + left.qualifiers, + )) + } U::Sort { keys, @@ -325,13 +513,16 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let keys = sort_keys(keys, &child.schema)?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - node(NonASAPOp::Sort { - keys, - partition_by, + let keys = sort_keys(keys, child.scope())?; + let partition_by = resolve_group_keys(partition_by, child.scope())?; + passing( + NonASAPOp::Sort { + keys, + partition_by, + child: Rc::clone(&child.node), + }, child, - }) + ) } U::Limit { @@ -341,35 +532,56 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - node(NonASAPOp::Limit { - n: *n, - offset: *offset, - partition_by, + let partition_by = resolve_group_keys(partition_by, child.scope())?; + passing( + NonASAPOp::Limit { + n: *n, + offset: *offset, + partition_by, + child: Rc::clone(&child.node), + }, child, - }) + ) } U::PromqlSubquery { range, resolution, child, - } => node(NonASAPOp::PromqlSubquery { - range: *range, - resolution: *resolution, - child: resolve(child, usage_derived_schema)?, - }), - - U::TimeRange { range, kind, child } => node(NonASAPOp::TimeRange { - range: *range, - kind: *kind, - child: resolve(child, usage_derived_schema)?, - }), - - U::TimeShift { shift, child } => node(NonASAPOp::TimeShift { - shift: *shift, - child: resolve(child, usage_derived_schema)?, - }), + } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::PromqlSubquery { + range: *range, + resolution: *resolution, + child: Rc::clone(&child.node), + }, + child, + ) + } + + U::TimeRange { range, kind, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::TimeRange { + range: *range, + kind: *kind, + child: Rc::clone(&child.node), + }, + child, + ) + } + + U::TimeShift { shift, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::TimeShift { + shift: *shift, + child: Rc::clone(&child.node), + }, + child, + ) + } U::SQLWindowFunc { func, @@ -383,19 +595,27 @@ fn resolve( let child = resolve(child, usage_derived_schema)?; let args = args .iter() - .map(|a| expr(a, &child.schema)) + .map(|a| expr(a, child.scope())) .collect::, _>>()?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - let order_by = sort_keys(order_by, &child.schema)?; - node(NonASAPOp::SQLWindowFunc { - func: func.clone(), - args, - partition_by, - order_by, - frame: frame.clone(), - output_name: output_name.clone(), - child, - }) + let partition_by = resolve_group_keys(partition_by, child.scope())?; + let order_by = sort_keys(order_by, child.scope())?; + // The window function's own column comes last, unqualified. + let mut qualifiers = child.qualifiers.clone(); + if !qualifiers.is_empty() { + qualifiers.push(None); + } + Ok(Bound::new( + node(NonASAPOp::SQLWindowFunc { + func: func.clone(), + args, + partition_by, + order_by, + frame: frame.clone(), + output_name: output_name.clone(), + child: child.node, + })?, + qualifiers, + )) } U::BinaryOp { @@ -414,11 +634,11 @@ fn resolve( .into_iter() .filter(|n| !own.contains(n)) .collect(); - node(NonASAPOp::BinaryOp { + unqualified(NonASAPOp::BinaryOp { operator: operator.clone(), return_bool: *return_bool, - lhs: resolve_root_with_inherited(lhs, &inherited)?, - rhs: resolve_root_with_inherited(rhs, &inherited)?, + lhs: resolve_root_with_inherited(lhs, &inherited)?.node, + rhs: resolve_root_with_inherited(rhs, &inherited)?.node, }) } } @@ -442,28 +662,29 @@ pub(crate) fn resolve_expr( expr: &UnresolvedScalar, schema: &Schema, ) -> Result { - resolve_expr_in(expr, schema, schema) + resolve_expr_in(expr, Scope::unqualified(schema), schema) } -/// [`resolve_expr`] where the operators the expression reads inherit from -/// `enclosing` (the owning root's usage-derived schema) rather than from `schema`. +/// [`resolve_expr`] in `scope`, where the operators the expression reads +/// inherit from `enclosing` (the owning root's usage-derived schema) rather +/// than from the scope's schema. fn resolve_expr_in( expr: &UnresolvedScalar, - schema: &Schema, + scope: Scope, enclosing: &Schema, ) -> Result { use UnresolvedScalar as S; let bx = |e: &UnresolvedScalar| -> Result, ResolveDAGError> { - Ok(Box::new(resolve_expr_in(e, schema, enclosing)?)) + Ok(Box::new(resolve_expr_in(e, scope, enclosing)?)) }; let each = |es: &[UnresolvedScalar]| -> Result, ResolveDAGError> { es.iter() - .map(|e| resolve_expr_in(e, schema, enclosing)) + .map(|e| resolve_expr_in(e, scope, enclosing)) .collect() }; let op = |o: &UnresolvedOp| resolve_nested_root(o, enclosing); Ok(match expr { - S::Column(c) => ScalarExpr::Column(resolve_column_ref(c, schema)?), + S::Column(c) => ScalarExpr::Column(scope.resolve(c)?), S::Literal(s) => ScalarExpr::Literal(s.clone()), S::EvalTimestamp => ScalarExpr::EvalTimestamp, S::CurrentTimestamp => ScalarExpr::CurrentTimestamp, @@ -492,7 +713,7 @@ fn resolve_expr_in( // nanosecond timestamp, which is the same IR type. S::Cast { expr, to, try_cast } => { let operand = bx(expr)?; - if matches!(operand.scalar_type(schema), Ok((dtype, _)) if dtype == *to) { + if matches!(operand.scalar_type(scope.schema), Ok((dtype, _)) if dtype == *to) { *operand } else { ScalarExpr::Cast { @@ -536,8 +757,8 @@ fn resolve_expr_in( .iter() .map(|(w, t)| { Ok(( - resolve_expr_in(w, schema, enclosing)?, - resolve_expr_in(t, schema, enclosing)?, + resolve_expr_in(w, scope, enclosing)?, + resolve_expr_in(t, scope, enclosing)?, )) }) .collect::, ResolveDAGError>>()?, @@ -564,9 +785,9 @@ fn resolve_expr_in( /// Resolve name-based group keys positionally, preserving `by`/`without`. fn resolve_group_keys( keys: &GroupKeys, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { - let ids = resolve_column_refs(keys.keys(), schema)?; + let ids = scope.resolve_all(keys.keys())?; Ok(if keys.is_without() { GroupKeys::without(ids) } else { @@ -587,15 +808,15 @@ fn resolve_group_keys( /// may exist at runtime, and PromQL leaves seed every referenced label via the /// schema resolver, so an unresolved key over an open schema indicates a resolution bug, /// not an absent label. (SQL is unaffected — its `GROUP BY` resolves through -/// the strict [`resolve_column_refs`], and DataFusion has already validated +/// the strict [`resolve_group_keys`], and DataFusion has already validated /// the columns anyway.) fn resolve_group_keys_promql( cols: &[ColumnRef], - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { cols.iter() - .filter_map(|c| match resolve_column_ref(c, schema) { - Err(ResolveError::NotFound { .. }) if schema.closed => None, + .filter_map(|c| match scope.resolve(c) { + Err(ResolveError::NotFound { .. }) if scope.schema.closed => None, other => Some(other), }) .collect() @@ -610,11 +831,11 @@ fn resolve_group_keys_promql( /// path is a no-op difference there. fn resolve_reduction( reduction: &Reduction, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { Ok(match reduction { Reduction::Reduce(by) => { - let ids = resolve_group_keys_promql(by.keys(), schema)?; + let ids = resolve_group_keys_promql(by.keys(), scope)?; Reduction::Reduce(if by.is_without() { GroupKeys::without(ids) } else { @@ -630,18 +851,16 @@ fn resolve_reduction( /// convention); every other field carries through unchanged. fn resolve_agg_intent( intent: &AggIntent, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { let col = |c: &Option| -> Result, ResolveError> { - c.as_ref() - .map(|r| resolve_column_ref(r, schema)) - .transpose() + c.as_ref().map(|r| scope.resolve(r)).transpose() }; Ok(match intent { AggIntent::Count => AggIntent::Count, AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { - left: resolve_column_ref(left, schema)?, - right: resolve_column_ref(right, schema)?, + left: scope.resolve(left)?, + right: scope.resolve(right)?, }, AggIntent::Sum { col: c } => AggIntent::Sum { col: col(c)? }, AggIntent::Min { col: c } => AggIntent::Min { col: col(c)? }, @@ -661,10 +880,7 @@ fn resolve_agg_intent( }, AggIntent::TopK { k } => AggIntent::TopK { k: *k }, AggIntent::Cardinality { cols } => AggIntent::Cardinality { - cols: cols - .iter() - .map(|c| resolve_column_ref(c, schema)) - .collect::>()?, + cols: scope.resolve_all(cols)?, }, AggIntent::FrequencyL2 { col: c } => AggIntent::FrequencyL2 { col: col(c)? }, AggIntent::FrequencyEntropy { col: c } => AggIntent::FrequencyEntropy { col: col(c)? }, @@ -692,7 +908,7 @@ fn resolve_agg_intent( }, AggIntent::HistogramQuantile { q, le } => AggIntent::HistogramQuantile { q: *q, - le: resolve_column_ref(le, schema)?, + le: scope.resolve(le)?, }, AggIntent::Math(f) => AggIntent::Math(f.clone()), AggIntent::Absent { labels } => AggIntent::Absent { @@ -832,11 +1048,7 @@ fn project_sample( } }) .collect(); - node(NonASAPOp::Project { - child, - cols, - qualifier: None, - }) + node(NonASAPOp::Project { child, cols }) } #[cfg(test)] @@ -859,6 +1071,7 @@ mod tests { }, predicates: vec![], schema: None, + qualifier: None, } } @@ -888,9 +1101,14 @@ mod tests { #[test] fn resolve_pearson_corr_inputs() { let schema = Schema::new(vec![ - Field::plain("x", DataType::Float64, true).with_table("a"), - Field::plain("x", DataType::Float64, true).with_table("b"), + Field::plain("x", DataType::Float64, true), + Field::plain("x", DataType::Float64, true), ]); + let qualifiers = [Some("a".to_string()), Some("b".to_string())]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let intent = AggIntent::PearsonCorr { left: ColumnRef::Qualified { table: "a".into(), @@ -902,7 +1120,7 @@ mod tests { }, }; assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), + resolve_agg_intent(&intent, scope).unwrap(), AggIntent::PearsonCorr { left: 0, right: 1 } ); let missing = AggIntent::PearsonCorr { @@ -912,7 +1130,7 @@ mod tests { }, right: ColumnRef::Named("missing".into()), }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); + assert!(resolve_agg_intent(&missing, scope).is_err()); } // Every leg resolves independently, qualifiers included; one unknown leg @@ -920,9 +1138,14 @@ mod tests { #[test] fn resolve_distinct_tuple_columns() { let schema = Schema::new(vec![ - Field::plain("k", DataType::Int64, true).with_table("a"), - Field::plain("k", DataType::Int64, true).with_table("b"), + Field::plain("k", DataType::Int64, true), + Field::plain("k", DataType::Int64, true), ]); + let qualifiers = [Some("a".to_string()), Some("b".to_string())]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let qualified = |table: &str| ColumnRef::Qualified { table: table.into(), name: "k".into(), @@ -931,13 +1154,13 @@ mod tests { cols: vec![qualified("b"), qualified("a")], }; assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), + resolve_agg_intent(&intent, scope).unwrap(), AggIntent::Cardinality { cols: vec![1, 0] } ); let missing = AggIntent::Cardinality { cols: vec![qualified("a"), ColumnRef::Named("missing".into())], }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); + assert!(resolve_agg_intent(&missing, scope).is_err()); } // ` > `: the bridged literal comes through unchanged, the @@ -1028,6 +1251,7 @@ mod tests { }, predicates: vec![], schema: Some(input), + qualifier: None, }), }; let resolved = resolve_root(&unresolved).expect("resolves"); @@ -1051,9 +1275,8 @@ mod tests { table_ref: table.into(), }, predicates: vec![], - schema: Some(Schema::new(vec![ - Field::plain("k", DataType::Int64, false).with_table(table) - ])), + schema: Some(Schema::new(vec![Field::plain("k", DataType::Int64, false)])), + qualifier: Some(table.into()), }; let qualified = |table: &str| { UnresolvedScalar::Column(ColumnRef::Qualified { @@ -1093,6 +1316,7 @@ mod tests { source: Source::TimeSeries { metric: "x".into() }, predicates: vec![UnresolvedPredicate(eq_lit(named("a"), "1"))], schema: None, + qualifier: None, }; let unresolved = UnresolvedOp::PromqlScalarOp { child: Rc::new(scan("m")), @@ -1182,7 +1406,7 @@ mod tests { closed: true, }; assert_eq!( - resolve_group_keys_promql(&[ColumnRef::Named("job".into())], &s), + resolve_group_keys_promql(&[ColumnRef::Named("job".into())], Scope::unqualified(&s)), Ok(vec![]) ); // Present keys still resolve positionally; absent ones drop around them. @@ -1192,7 +1416,7 @@ mod tests { ColumnRef::Named("job".into()), ColumnRef::Named("group".into()) ], - &s + Scope::unqualified(&s) ), Ok(vec![0]) ); @@ -1205,7 +1429,7 @@ mod tests { // absent label. Keep the strict error. let open = ts_value_schema(); // closed: false assert!(matches!( - resolve_group_keys_promql(&[ColumnRef::Named("job".into())], &open), + resolve_group_keys_promql(&[ColumnRef::Named("job".into())], Scope::unqualified(&open)), Err(ResolveError::NotFound { .. }) )); } diff --git a/crates/frontend-common/src/schema_resolver.rs b/crates/frontend-common/src/schema_resolver.rs index 5462304cc..96be18f6e 100644 --- a/crates/frontend-common/src/schema_resolver.rs +++ b/crates/frontend-common/src/schema_resolver.rs @@ -222,6 +222,7 @@ mod tests { }, predicates: vec![], schema: None, + qualifier: None, } } diff --git a/crates/frontend-common/src/unresolved.rs b/crates/frontend-common/src/unresolved.rs index 52255beeb..0e55023d4 100644 --- a/crates/frontend-common/src/unresolved.rs +++ b/crates/frontend-common/src/unresolved.rs @@ -114,6 +114,10 @@ pub enum UnresolvedOp { /// `Some` for a catalog-backed (SQL) leaf; `None` defers to the /// usage-derived schema resolver. schema: Option, + /// The SQL table or alias qualifying the leaf's columns, which + /// qualified references (`t.col`) above resolve by; `None` for a + /// PromQL leaf. + qualifier: Option, }, Values { rows: Vec>, @@ -125,6 +129,9 @@ pub enum UnresolvedOp { }, Project { cols: Vec, + /// The alias of a derived table (`FROM (SELECT …) t`) qualifying + /// every output column, which qualified references above resolve + /// by; `None` for an ordinary SELECT list. qualifier: Option, child: Rc, }, diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs index 23df96169..4c38d3ce7 100644 --- a/crates/frontend-metricsql/src/lib.rs +++ b/crates/frontend-metricsql/src/lib.rs @@ -153,6 +153,7 @@ impl Lowerer<'_> { .map(|f| UnresolvedPredicate(matcher(f))) .collect(), schema: None, + qualifier: None, }) } diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 942593d70..731d3b0c3 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -1671,6 +1671,7 @@ fn filtered_source(metric: String, matchers: Vec, shift: TimeShift) -> U predicates: matchers.into_iter().map(UnresolvedPredicate).collect(), // Usage-derived (PromQL is schemaless) — the schema resolver fills this in. schema: None, + qualifier: None, }; if shift.is_identity() { scan diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs index 755e64514..3c4a044a9 100644 --- a/crates/frontend-sql/src/frequency.rs +++ b/crates/frontend-sql/src/frequency.rs @@ -336,7 +336,7 @@ fn sql_frequency_result( name: &str, value: ScalarExpr, ) -> Option> { - let NonASAPOp::Project { qualifier, .. } = root.non_asap()? else { + let NonASAPOp::Project { .. } = root.non_asap()? else { return None; }; let aggregate = |measure, name: &str| { @@ -380,7 +380,6 @@ fn sql_frequency_result( else_expr: Some(Box::new(value)), }, }], - qualifier: qualifier.clone(), child, })) .ok()?; diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs index b015c0032..09edf5759 100644 --- a/crates/frontend-sql/src/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -528,7 +528,7 @@ impl<'a> SqlLowerer<'a> { self.scan_source(&table, &table) } - /// A `Scan` over catalog table `table`, with its columns qualified by + /// A `Scan` over catalog table `table`, its columns qualified by /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so /// `Qualified` column refs resolve to the right side across a join. fn scan_source(&self, table: &str, qualifier: &str) -> Result { @@ -537,24 +537,17 @@ impl<'a> SqlLowerer<'a> { .tables .get(table) .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; - let qualified = Schema { - fields: schema - .fields - .iter() - .cloned() - .map(|c| c.with_table(qualifier)) - .collect(), - time_index: schema.time_index, - unique_keys: schema.unique_keys.clone(), - // Catalog-backed: the table's columns are fully declared → closed. - closed: true, - }; Ok(Unresolved::Scan { source: Source::Table { table_ref: table.to_string(), }, predicates: vec![], - schema: Some(qualified), + schema: Some(Schema { + // Catalog-backed: the table's columns are fully declared → closed. + closed: true, + ..schema.clone() + }), + qualifier: Some(qualifier.to_string()), }) } @@ -1929,12 +1922,14 @@ fn filter_or_fold(pred: Scalar, child: Unresolved) -> Unresolved { source, mut predicates, schema, + qualifier, } => { predicates.push(Predicate(pred)); Unresolved::Scan { source, predicates, schema, + qualifier, } } other => Unresolved::Filter { diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 4ae70b9f9..6650aec43 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -2865,6 +2865,34 @@ async fn lower_err(sql: &str) -> String { .to_string() } +/// Both join sides have a `service` column; the front end binds `h.service` +/// with its own scope while lowering, so the group key is the right side's +/// column, the IR schema carrying no qualifier. +#[tokio::test] +async fn a_qualified_group_key_binds_its_join_side() { + let qe = lower( + "SELECT h.service, SUM(m.bytes) FROM metrics m JOIN hosts h ON m.service = h.service \ + GROUP BY h.service", + ) + .await; + let aggregate = find_aggregate_node(&qe).expect("aggregate"); + let NonASAPOp::Aggregate { + reduction, child, .. + } = op(aggregate) + else { + unreachable!() + }; + let [key] = reduction.expect_reduce().keys() else { + panic!("one key: {reduction:?}") + }; + let metrics_columns = 4; + assert!( + *key >= metrics_columns, + "{key} is a metrics column: {child:#?}" + ); + assert_eq!(child.schema.fields[*key].name, "service"); +} + /// After a join the `Project` drops table qualifiers, so `h.service` must not /// fall back to the first bare `service` (`m.service`): the reference is /// ambiguous and rejected rather than sorting by the wrong column. @@ -2875,7 +2903,7 @@ async fn qualified_order_by_after_a_join_does_not_bind_the_wrong_column() { ORDER BY h.service", ) .await; - assert!(err.contains("several unqualified columns"), "{err}"); + assert!(err.contains("several columns"), "{err}"); } /// `WITHIN GROUP (ORDER BY x DESC)` reads the percentile from the top, as diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index b58aa2ec4..314601b2b 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -57,8 +57,9 @@ pub use asap_stage3_plan_selection::{ AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, }; pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; +pub use asap_types::ir::operator::arguments::SourceColumn; pub use asap_types::ir::properties::selection::{ - ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, + Constraint, IntervalConstraintLimit, SelectionBox, }; pub use asap_types::ir::properties::ExecutionTiming; pub use asap_types::plan::materialized_view::{ diff --git a/crates/types/src/accuracy/target.rs b/crates/types/src/accuracy/target.rs index e460ae9af..1b3562729 100644 --- a/crates/types/src/accuracy/target.rs +++ b/crates/types/src/accuracy/target.rs @@ -704,7 +704,6 @@ mod tests { else_expr: Some(Box::new(otherwise)), }, }], - qualifier: None, child: joined.clone(), })) }; diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index e3f090294..35d1be914 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -129,7 +129,6 @@ mod tests { alias: Some("s".into()), expr: ScalarExpr::ScalarSubquery(Rc::clone(&v)), }], - qualifier: None, child: Rc::clone(&v), })) .unwrap(); diff --git a/crates/types/src/ir/operator/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs index 799ce1ade..2449391bf 100644 --- a/crates/types/src/ir/operator/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -499,9 +499,11 @@ impl AggIntent { ), // Ranked rows retain the ranked value's type; the enclosing // operator also preserves item identity via schema::topk. - AggIntent::TopK { k, .. } => { - col(&format!("topk_{k}"), input.dtype.clone(), input.nullable) - } + AggIntent::TopK { k, .. } => col( + &crate::ir::schema::topk::topk_output_name(*k, &[]), + input.dtype.clone(), + input.nullable, + ), AggIntent::Cardinality { .. } => col("cardinality", DataType::Int64, false), AggIntent::FrequencyL2 { .. } => col("frequency_l2", DataType::Float64, false), AggIntent::FrequencyEntropy { .. } => { diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index 2b6ff8f78..57b7cd2c7 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -178,6 +178,17 @@ pub enum Source { Table { table_ref: String }, } +/// A column of a source (a `Scan`'s metric or table), by name: the lineage +/// of a column copied from it +/// (`ColumnLineage::SourceColumn`, a memo-group property in `crate::stages`), +/// and the key an aggregate coverage's selection names a column by. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceColumn { + pub source: Source, + pub column: String, +} + impl Source { /// Whether it is a time series (PromQL rows): a PromQL top-k over rows /// that scan only time series ranks one row per series. @@ -521,7 +532,7 @@ mod tests { use super::*; use crate::ir::schema::{DataType, Field, Schema}; - /// A key moves to the input's column of the same name and qualifier; a + /// A key moves to the input's column of the same name; a /// key that is missing or ambiguous there, or a `without` grouping, has no /// partitions over the input. #[test] @@ -542,11 +553,6 @@ mod tests { Reduction::by(vec![0]).partitions_over(&inner, &ambiguous), None ); - let qualified = Schema::new(vec![field("job").with_table("t")]); - assert_eq!( - Reduction::by(vec![0]).partitions_over(&inner, &qualified), - None - ); assert_eq!( Reduction::Reduce(GroupKeys::without(vec![0])).partitions_over(&inner, &input), None diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 72bc8ad8f..4f4a0a685 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -136,7 +136,7 @@ impl OperatorNode { Ok(Self::with_schema(operator, schema)) } - /// Build a node with caller-supplied output names and qualifiers. For + /// Build a node with caller-supplied output names. For /// either operator category, `validate_structure` requires all other /// schema metadata to agree with derivation; output kind is derived here. pub fn with_schema(operator: Operator, schema: Schema) -> Self { @@ -205,10 +205,9 @@ impl OperatorNode { /// Replace the children and rebuild: each child is replaced by `f(child)` /// and the node is rebuilt over the new children, re-deriving all - /// structural schema metadata. Only names and qualifiers that override - /// the old derived schema are retained, for either operator category. A - /// change in output arity with such overrides needs an explicit new - /// naming assignment. + /// structural schema metadata. Only names that override the old derived + /// schema are retained, for either operator category. A change in output + /// arity with such overrides needs an explicit new naming assignment. /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, @@ -218,22 +217,15 @@ impl OperatorNode { let mut rebuilt = Self::new(self.operator.map_children(f))?; for (i, (derived, retained)) in previous.fields.iter().zip(&self.schema.fields).enumerate() { - let renamed = retained.name != derived.name; - let requalified = retained.table != derived.table; - if !renamed && !requalified { + if retained.name == derived.name { continue; } if rebuilt.schema.fields.len() != previous.fields.len() { return Err(SchemaDerivationError::InvalidScalarSignature( - "output arity changed; reassign explicit output names and qualifiers".into(), + "output arity changed; reassign explicit output names".into(), )); } - if renamed { - rebuilt.schema.fields[i].name = retained.name.clone(); - } - if requalified { - rebuilt.schema.fields[i].table = retained.table.clone(); - } + rebuilt.schema.fields[i].name = retained.name.clone(); } Ok(rebuilt) } @@ -277,7 +269,7 @@ impl OperatorNode { /// input contract, scalar typing against the owning operator's input /// schema, and agreement between each retained schema and the one /// derived from the operator. Both operator categories may override field - /// names and qualifiers; all structural metadata must match derivation. + /// names; all structural metadata must match derivation. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { for node in Self::reachable(self) { if node.schema.time_index.is_some_and(|i| { @@ -306,7 +298,6 @@ impl OperatorNode { // so new structural metadata cannot accidentally escape validation. for (field, retained) in derived.fields.iter_mut().zip(&node.schema.fields) { field.name = retained.name.clone(); - field.table = retained.table.clone(); } let agree = derived == node.schema; if !agree { diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 2312fd73c..892ce65fb 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -75,14 +75,7 @@ pub enum NonASAPOp> { /// σ — row-level filter. Output schema = child schema. Filter { pred: Predicate, child: C }, /// π — projection. - Project { - cols: Vec>, - /// Re-qualifies every output column with this table alias (a derived - /// table / inline view). `None` for an ordinary SELECT list. - #[serde(default)] - qualifier: Option, - child: C, - }, + Project { cols: Vec>, child: C }, /// γ + α — grouping + aggregate intents. Aggregate { reduction: Reduction, @@ -300,11 +293,7 @@ impl NonASAPOp { child: f(child), } } - Project { - cols, - qualifier, - child, - } => { + Project { cols, child } => { let cols = cols .iter() .map(|c| ProjectItem { @@ -314,7 +303,6 @@ impl NonASAPOp { .collect(); Project { cols, - qualifier: qualifier.clone(), child: f(child), } } @@ -597,16 +585,11 @@ impl NonASAPOp { .. } => { if let [AggIntent::TopK { k, .. }] = measures.as_slice() { - let name = output_names - .first() - .filter(|name| !name.is_empty()) - .cloned() - .unwrap_or_else(|| format!("topk_{k}")); + let name = crate::ir::schema::topk::topk_output_name(*k, output_names); let cols = crate::ir::schema::topk::ranked_row_projection(child, reduction, &name)?; return NonASAPOp::Project { cols, - qualifier: None, child: child.clone(), } .output_schema(); @@ -695,11 +678,7 @@ impl NonASAPOp { // keeps its field verbatim, so an `ExactAggregate` state column // can pass through a projection unchanged; any other expression // is typed against the input and must read plain values. - Project { - cols, - qualifier, - child, - } => { + Project { cols, child } => { let in_schema = child.schema(); let fields: Vec = cols .iter() @@ -707,9 +686,7 @@ impl NonASAPOp { .map(|(i, item)| { let mut field = match &item.expr { ScalarExpr::Column(id) if in_schema.fields.get(*id).is_some() => { - let mut f = in_schema.fields[*id].clone(); - f.table = None; - f + in_schema.fields[*id].clone() } expr => { let (dtype, nullable) = expr.scalar_type(in_schema)?; @@ -720,10 +697,7 @@ impl NonASAPOp { .alias .clone() .unwrap_or_else(|| default_proj_name(&item.expr, i, in_schema)); - Ok(match qualifier { - Some(q) => field.with_table(q), - None => field, - }) + Ok(field) }) .collect::, SchemaDerivationError>>()?; let time_index = fields.iter().position(|c| c.name == "ts"); @@ -1428,7 +1402,6 @@ mod tests { fn project(cols: Vec, child: NonASAPOp) -> NonASAPOp { NonASAPOp::Project { cols, - qualifier: None, child: node(child), } } diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs index 2af73e527..4ce95601b 100644 --- a/crates/types/src/ir/properties/selection.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -1,7 +1,8 @@ //! The selection algebra of aggregate coverage (a memo group's coverage //! `selection`, in `crate::stages`): which output rows of a computation a //! state took, as a union of boxes ([`SelectionBox`]), each a -//! [`Constraint`] per column ([`ColumnIdentity`]): value sets and intervals, +//! [`Constraint`] per column, named by its source column ([`SourceColumn`]): +//! value sets and intervals, //! with interval limits relative to the evaluation time //! ([`IntervalConstraintLimit`]). A predicate conjunct becomes a constraint //! through `constraint_of`. @@ -11,6 +12,8 @@ use std::ops::Bound; use serde::{Deserialize, Serialize}; +use crate::ir::operator::arguments::SourceColumn; + use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::ScalarExpr; use crate::ir::schema::ColumnId; @@ -24,24 +27,24 @@ use crate::ir::schema::ColumnId; #[serde(deny_unknown_fields)] pub struct SelectionBox { #[serde(with = "columns_as_list")] - pub(crate) columns: BTreeMap, + pub(crate) columns: BTreeMap, } /// `SelectionBox::columns` as a list of `{column, constraint}`. mod columns_as_list { - use super::{ColumnIdentity, Constraint}; + use super::{Constraint, SourceColumn}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Entry { - column: ColumnIdentity, + column: SourceColumn, constraint: Constraint, } pub(super) fn serialize( - columns: &BTreeMap, + columns: &BTreeMap, serializer: S, ) -> Result { let entries: Vec = columns @@ -56,7 +59,7 @@ mod columns_as_list { pub(super) fn deserialize<'de, D: Deserializer<'de>>( deserializer: D, - ) -> Result, D::Error> { + ) -> Result, D::Error> { let entries = Vec::::deserialize(deserializer)?; Ok(entries .into_iter() @@ -94,26 +97,6 @@ impl From> for Bound { } } -/// A column of the definition child's output, by `(table, name)`. A -/// `Project` below renames its columns and sets their table to its -/// qualifier (none by default). -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ColumnIdentity { - pub(crate) table: Option, - pub(crate) name: String, -} - -#[cfg(any(test, feature = "test-support"))] -impl ColumnIdentity { - pub fn new(table: Option, name: impl Into) -> Self { - Self { - table, - name: name.into(), - } - } -} - /// A constraint on one column of a selection. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -463,12 +446,12 @@ impl Constraint { #[cfg(any(test, feature = "test-support"))] impl SelectionBox { - pub fn new(columns: BTreeMap) -> Self { + pub fn new(columns: BTreeMap) -> Self { Self { columns } } /// The per-column constraints. - pub fn columns(&self) -> &BTreeMap { + pub fn columns(&self) -> &BTreeMap { &self.columns } } @@ -566,6 +549,7 @@ fn touching( #[cfg(test)] mod tests { use super::*; + use crate::ir::operator::arguments::Source; use crate::ir::scalar::ExprSemantics; use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; @@ -590,7 +574,10 @@ mod tests { }; assert!(!last_5m.disjoint(&at)); assert_eq!(last_5m.intersect(&at), None); - let column = ColumnIdentity::new(None, "ts"); + let column = SourceColumn { + source: Source::TimeSeries { metric: "m".into() }, + column: "ts".into(), + }; let one = |constraint: Constraint| SelectionBox { columns: BTreeMap::from([(column.clone(), constraint)]), }; diff --git a/crates/types/src/ir/scalar/column_resolution.rs b/crates/types/src/ir/scalar/column_resolution.rs index 8822c6e85..e5bfc9d32 100644 --- a/crates/types/src/ir/scalar/column_resolution.rs +++ b/crates/types/src/ir/scalar/column_resolution.rs @@ -2,8 +2,9 @@ //! //! Front ends emit `ColumnRef` (name-based, optionally table-qualified); the //! IR uses positional [`ColumnId`] resolved against a per-node [`Schema`]. -//! These helpers turn name-based refs into positional ids, qualifier-aware. -//! Front-end name resolution (`asap_frontend_common::resolve`) calls them. +//! These helpers turn name-based refs into positional ids. A schema carries +//! no qualifier, so front-end name resolution (`asap_frontend_common::resolve`) +//! binds a qualified name with its own scope before it calls them. use thiserror::Error; @@ -22,38 +23,24 @@ pub enum ResolveError { NoSampleValue { available: Vec }, #[error("ColumnRef::Wildcard cannot be resolved to a single ColumnId")] WildcardNotPositional, - #[error("column `{name}` matches several unqualified columns")] + #[error("column `{name}` matches several columns")] Ambiguous { name: String }, } /// Resolve a single [`ColumnRef`] to a positional [`ColumnId`]. pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result { match col { - // A bare name must be unique: after a join `k` could be `a.k` or `b.k`. - ColumnRef::Named(name) => match schema.fields.iter().filter(|c| &c.name == name).count() { - 0 => Err(ResolveError::NotFound { - name: name.clone(), - available: schema.fields.iter().map(|c| c.name.clone()).collect(), - }), - 1 => Ok(schema.column_id(name).expect("one field has this name")), - _ => Err(ResolveError::Ambiguous { name: name.clone() }), - }, - // Prefer the (table, name) match; fall back to the bare name for - // schemas whose columns carry no qualifier, but only when that name - // is unique: after a join `b.k` must not bind to `a.k`. - ColumnRef::Qualified { table, name } => { - if let Some(id) = schema.column_id_qualified(table, name) { - return Ok(id); - } + // A name must be unique: after a join `k` could be either side's. + // A schema carries no qualifier; the front end binds a qualified + // name with its own scope before it calls this. + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => { match schema.fields.iter().filter(|c| &c.name == name).count() { 0 => Err(ResolveError::NotFound { - name: format!("{table}.{name}"), + name: name.clone(), available: schema.fields.iter().map(|c| c.name.clone()).collect(), }), 1 => Ok(schema.column_id(name).expect("one field has this name")), - _ => Err(ResolveError::Ambiguous { - name: format!("{table}.{name}"), - }), + _ => Err(ResolveError::Ambiguous { name: name.clone() }), } } ColumnRef::SampleValue => schema diff --git a/crates/types/src/ir/scalar/expr_ir.rs b/crates/types/src/ir/scalar/expr_ir.rs index 3ee1dd56d..f66493338 100644 --- a/crates/types/src/ir/scalar/expr_ir.rs +++ b/crates/types/src/ir/scalar/expr_ir.rs @@ -18,9 +18,10 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ColumnRef { Named(String), - /// Table-qualified reference (`t.col` / `alias.col`). Resolved by - /// `(table, name)` so a column name shared across a join (`a.k` vs `b.k`) - /// binds to the correct side. + /// Table-qualified reference (`t.col` / `alias.col`), as a front end + /// emits it. The front end binds it by `(table, name)` with its own + /// scope while it lowers, so a column name shared across a join (`a.k` + /// vs `b.k`) binds to the correct side; a schema carries no qualifier. Qualified { table: String, name: String, diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index d06c82d00..ce7644b65 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -905,7 +905,6 @@ mod tests { alias: Some("result".into()), expr, }], - qualifier: None, child: OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { table_ref: "t".into(), diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 72e3e5627..a6f65a4cd 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -36,8 +36,10 @@ use crate::summary_type::{ /// joins, and does not own data or prescribe a row/column-oriented storage layout. pub type ColumnId = usize; -/// One field of a [`Schema`]: `name + dtype + nullable`, plus an optional -/// table qualifier. The struct describes a column and holds none of its data. +/// One field of a [`Schema`]: `name + dtype + nullable`. The struct describes +/// a column and holds none of its data. Where a column's values come from is +/// its memo group's column lineage, not part of the field; a front end binds +/// qualified names (`t.col`) with its own scope while it lowers. /// /// `T` is the type vocabulary: [`FieldDataType`] on an operator edge (the /// default, and what [`Schema::fields`] holds), plain [`DataType`] for the @@ -53,46 +55,25 @@ pub struct Field { /// Whether NULL values are allowed in this field. PromQL value /// columns are non-nullable; SQL columns inherit their DDL nullability. pub nullable: bool, - /// Optional table/alias qualifier (SQL `t.col` / `t AS a` → `a`). Travels - /// with the field through joins so a `ColumnRef::Qualified` can pick the - /// right side when both carry the same `name`. `None` for PromQL labels and - /// unqualified columns. - #[serde(default)] - pub table: Option, } impl Field { - /// An unqualified field (`table = None`). pub fn new(name: impl Into, dtype: T, nullable: bool) -> Self { Self { name: name.into(), dtype, nullable, - table: None, } } - /// This field re-qualified under `table` (e.g. by a `SubqueryAlias`). - pub fn with_table(mut self, table: impl Into) -> Self { - self.table = Some(table.into()); - self - } - - /// A reference to this field: qualified by its table when it has one. + /// A reference to this field by its name. pub fn column_ref(&self) -> crate::ir::scalar::ColumnRef { - use crate::ir::scalar::ColumnRef; - match &self.table { - Some(table) => ColumnRef::Qualified { - table: table.clone(), - name: self.name.clone(), - }, - None => ColumnRef::Named(self.name.clone()), - } + crate::ir::scalar::ColumnRef::Named(self.name.clone()) } } impl Field { - /// An unqualified field carrying an ordinary readable value. + /// A field carrying an ordinary readable value. pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self { Self::new(name, FieldDataType::Plain(dtype), nullable) } @@ -129,7 +110,6 @@ impl From> for Field { name: field.name, dtype: FieldDataType::Plain(field.dtype), nullable: field.nullable, - table: field.table, } } } @@ -240,7 +220,7 @@ pub enum DataType { /// catalog tables. Date, /// Variable-length sequence. The existing column contract preserves the - /// element field name, type, and nullability. Nested fields are unqualified. + /// element field name, type, and nullability. List { element: Box> }, /// Ordered named fields, including each field's independent nullability. Struct { fields: Vec> }, @@ -305,7 +285,6 @@ impl Schema { field.name == PROMQL_SERIES_IDENTITY && field.dtype == DataType::Utf8 && !field.nullable - && field.table.is_none() }) } @@ -358,15 +337,6 @@ impl Schema { self.fields.iter().position(|c| c.name == name) } - /// Look up a field by `(table, name)` qualifier — disambiguates columns - /// that share a `name` across a join (`a.k` vs `b.k`). `None` if no field - /// has both that qualifier and name. - pub(crate) fn column_id_qualified(&self, table: &str, name: &str) -> Option { - self.fields - .iter() - .position(|c| c.name == name && c.table.as_deref() == Some(table)) - } - /// Whether this schema has *any* provable unique key. #[cfg(any(test, feature = "test-support"))] pub fn has_unique_key(&self) -> bool { @@ -447,12 +417,4 @@ mod tests { fn schema_without_fields_is_rejected() { assert!(serde_json::from_str::(r#"{"closed":true}"#).is_err()); } - - #[test] - fn qualified_field_serde_roundtrip() { - let c = col("service", DataType::Utf8).with_table("hosts"); - let back: Field = serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap(); - assert_eq!(back, c); - assert_eq!(back.table.as_deref(), Some("hosts")); - } } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 3ad853e70..685e1aa5b 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -56,6 +56,16 @@ pub fn ranked_column(child: &I) -> Result String { + output_names + .first() + .filter(|name| !name.is_empty()) + .cloned() + .unwrap_or_else(|| format!("topk_{k}")) +} + /// Ordered projection of ranked rows: partition keys, ranked value, then /// item identity. TopK retains rows; partition keys alone are not unique. pub(crate) fn ranked_row_projection( diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index 0cbd925a9..a05b93f79 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -13,8 +13,10 @@ //! predicate conjunct moves into `selection` when it can be lifted to the //! top (through `Filter`, a range `TimeRange`, a `TimeShift` and //! direct-column `Project` items) and it is a value set or an interval on -//! one column. The `TimeRange` and `TimeShift`s become one interval on the -//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! one column whose lineage is a source column: a selection names the +//! column by that [`SourceColumn`] (the input groups' +//! `GroupProperties::column_lineage`). The `TimeRange` and `TimeShift`s +//! become one interval on the timestamp column ([`IntervalConstraintLimit`]). Everything else stays in //! `definition` as a residual, so two states are merged only when they //! compute the same thing over disjoint rows. @@ -23,18 +25,19 @@ use std::ops::Bound; use thiserror::Error; +use super::column_properties::ColumnLineage; use super::group::MemoGroupAsInput; -use crate::ir::operator::arguments::{AtModifier, Reduction}; +use crate::ir::operator::arguments::{AtModifier, Reduction, SourceColumn}; use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; use crate::ir::properties::selection::{ - constraint_of, union, ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, + constraint_of, union, Constraint, IntervalConstraintLimit, SelectionBox, }; use crate::ir::scalar::expr_ir::ScalarValue; use crate::ir::scalar::{Predicate, ScalarExpr}; -use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::schema::ColumnId; use crate::ir::SchemaDerivationError; use crate::stages::MemoGroupId; use crate::summary_type::{MergeInputRule, SummaryTypeCapability}; @@ -271,7 +274,7 @@ fn selected_build<'m>( /// rows with timestamp column `time_column`. fn of_merge( inputs: &[AggregateCoverage], - time_column: Option<&ColumnIdentity>, + time_column: Option<&SourceColumn>, ) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; if inputs @@ -344,9 +347,11 @@ fn regrouped( if !SummaryTypeCapability::merges_groups(family) { return Err(CoverageError::SummaryTypeCannotRegroup); } - let state = first.schema(); - let rows = build_input_schema(first).ok_or(CoverageError::NoCoverage)?; - // The build's input column each key of the state names. + let state = first.group_properties(); + let rows = build_input(first).ok_or(CoverageError::NoCoverage)?; + let rows_lineage = &rows.group_properties().column_lineage; + // The build's input column each key of the state names: the column of + // the key's lineage. let mapped = keys .iter() .map(|&key| { @@ -358,20 +363,19 @@ fn regrouped( }; column .filter(|&column| { - let (Some(at_state), Some(at_input)) = - (state.fields.get(key), rows.fields.get(column)) - else { - return false; - }; - at_state.is_plain() - && at_state.name == at_input.name - && at_state.table == at_input.table + state + .schema + .fields + .get(key) + .is_some_and(|field| field.is_plain()) + && state.column_lineage.get(key).is_some() + && state.column_lineage.get(key) == rows_lineage.get(column) }) .ok_or(CoverageError::NotAGroupKey { key }) }) .collect::, _>>()?; let reduction = Reduction::by(mapped); - summary_agg_layout(&rows, family, &reduction).map_err(|error| { + summary_agg_layout(rows.schema(), family, &reduction).map_err(|error| { CoverageError::RegroupedBuild { reason: error.to_string(), } @@ -391,30 +395,31 @@ fn regrouped( }) } -/// The schema of the rows the build of summary state `state` reads (its +/// The input group the build of summary state `state` reads (its /// `SummaryAgg`'s input; one build over a chain's window reads its range of -/// the chain's input, which has the input's schema). `None` when `state` -/// is no summary state. -fn build_input_schema(state: &MemoGroupAsInput<'_>) -> Option { +/// the chain's input, which has the input's schema and lineage). `None` +/// when `state` is no summary state. +fn build_input<'m>(state: &MemoGroupAsInput<'m>) -> Option> { if let Some(states) = selected_from(state) { - return build_input_schema(&states); + return build_input(&states); } match state.operator() { Operator::ASAP( ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => Some(child.schema().clone()), - Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { - build_input_schema(children.first()?) - } + ) => Some(child), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => build_input(children.first()?), _ => None, } } /// The timestamp column of the rows the build of summary state `state` -/// reads ([`build_input_schema`]), as a selection names it. -fn time_column_of_build_input(state: &MemoGroupAsInput<'_>) -> Option { - let rows = build_input_schema(state)?; - column_identity(&rows, rows.time_index?) +/// reads ([`build_input`]), as a selection names it. +fn time_column_of_build_input(state: &MemoGroupAsInput<'_>) -> Option { + let rows = build_input(state)?; + source_column( + &rows.group_properties().column_lineage, + rows.schema().time_index?, + ) } /// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` @@ -486,7 +491,8 @@ fn of_aggregate<'m>( let InputOrTemporaryBuild::Input(base_input) = &base else { unreachable!("a rule builds only liftable operators over real inputs"); }; - let top = child.schema(); + let top = lineage_at_top(&base_input.group_properties().column_lineage, &chain); + let top = top.as_slice(); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. let mut lifted = BTreeMap::new(); @@ -533,7 +539,7 @@ fn of_aggregate<'m>( .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; let column = chain[depth].children()[0].schema().time_index?; let column = column_at_top(column, &chain[..=depth])?; - column_identity(top, column)?; + source_column(top, column)?; Some((column, combined_with(lifted.get(&column), constraint)?)) }); let lift_time = time.is_some(); @@ -566,7 +572,7 @@ fn of_aggregate<'m>( let selection = SelectionBox { columns: lifted .into_iter() - .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) + .filter_map(|(column, constraint)| Some((source_column(top, column)?, constraint))) .collect(), }; Ok(AggregateCoverage { @@ -579,13 +585,13 @@ fn of_aggregate<'m>( } /// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest, each scalar subquery +/// the agg child, whose lineage is `top`) and return the rest, each scalar subquery /// naming the input it reads. `above` /// are the operators between the predicate and the `SummaryAgg`, top to -/// bottom. A column [`column_identity`] cannot name stays. +/// bottom. A conjunct on a column [`source_column`] cannot name stays. fn residual( pred: &Predicate>, - top: &Schema, + top: &[ColumnLineage], above: &[Operator>], lifted: &mut BTreeMap, ) -> Option>> { @@ -593,7 +599,7 @@ fn residual( for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; - column_identity(top, column)?; + source_column(top, column)?; Some((column, combined_with(lifted.get(&column), constraint)?)) }); match lift { @@ -649,19 +655,44 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option Option { - let field = schema.fields.get(column)?; - let namesakes = schema - .fields +/// The lineage of the aggregate's input columns: the lineage of `base` +/// (the group below the liftable `chain`, top down), carried up through +/// it: a `Filter`, `TimeRange` or `TimeShift` keeps its input's, a +/// `Project`'s column item copies its input column's and an expression item +/// is `Computed`. A rule's temporary build in the chain needs no lineage of +/// its own. +fn lineage_at_top(base: &[ColumnLineage], chain: &[Operator]) -> Vec { + let mut lineage = base.to_vec(); + for link in chain.iter().rev() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { + lineage = cols + .iter() + .map(|item| match item.expr { + ScalarExpr::Column(column) => lineage + .get(column) + .cloned() + .unwrap_or(ColumnLineage::Computed), + _ => ColumnLineage::Computed, + }) + .collect(); + } + } + lineage +} + +/// The source column a selection names column `column` by: its lineage, one +/// of `lineage` (per column). `None` when the column is `Computed`, or +/// another column has the same lineage (`Project [a AS k, a AS k2]`), so a +/// selection cannot name it apart. +fn source_column(lineage: &[ColumnLineage], column: ColumnId) -> Option { + let ColumnLineage::SourceColumn(source_column) = lineage.get(column)? else { + return None; + }; + let namesakes = lineage .iter() - .filter(|f| f.name == field.name && f.table == field.table) + .filter(|other| *other == &lineage[column]) .count(); - (namesakes == 1).then(|| ColumnIdentity { - table: field.table.clone(), - name: field.name.clone(), - }) + (namesakes == 1).then(|| source_column.clone()) } /// The interval on the timestamp column that `links` (top down, the diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index 23598bd52..e2d4efe16 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -1,18 +1,26 @@ -//! Value sign: the sign of each output column of a memo group (concepts -//! doc, section 3, "Value sign"), derived once from the input groups' signs -//! and, for a `Scan`, the signs the data workload declares for its source's -//! columns. Error contracts that hold only for values of one sign read it. +//! Column lineage and value sign: two per-column group properties +//! (concepts doc, section 3, "Column lineage" and "Value sign"), derived +//! together, once per group, from the input groups' and, for a `Scan`, from +//! its source's columns and the signs the data workload declares for them. +//! One per-operator column layout says which input column each output +//! column copies (lineage and sign kept) and which it computes (lineage +//! `Computed`, the sign by the operator's rule). Error contracts that hold +//! only for values of one sign read the sign; aggregate coverage names a +//! selection's columns by the lineage. use std::collections::BTreeMap; use super::group::MemoGroupAsInput; -use crate::ir::operator::arguments::{BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction}; +use crate::ir::operator::arguments::{ + BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, SourceColumn, +}; use crate::ir::operator::asap::{summary_agg_layout, summary_build}; use crate::ir::operator::input::OperatorInput; use crate::ir::operator::AggIntent; use crate::ir::scalar::expr_ir::{ArithmeticOpKind, ScalarValue}; use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; +use crate::ir::schema::topk::{ranked_row_projection, topk_output_name}; use crate::ir::schema::{FieldDataType, Schema}; use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ @@ -42,71 +50,137 @@ pub enum ValueSign { Unknown, } +/// Where an output column's values come from, a group property derived +/// from the input groups (concepts doc, section 3, "Column lineage"). It +/// names a column across groups where a schema field's name cannot: two +/// join sides' columns of one name have different lineage, and a renamed +/// column keeps its lineage. Aggregate coverage names a selection's column +/// by it. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum ColumnLineage { + /// Copied unchanged from a column of a source. + SourceColumn(SourceColumn), + /// Computed by an operator: a measure, an expression, a summary state or + /// a readout. + Computed, +} + use ValueSign::{NonNegative, NonPositive, Unknown}; type Input<'m> = MemoGroupAsInput<'m>; -/// The sign of each column of `output`, the output schema of `operator`, -/// from its input groups' `column_signs` and, for a `Scan`, `source_facts`. -pub(super) fn column_signs( +/// One output column's lineage and sign. +#[derive(Debug, Clone, PartialEq)] +struct ColumnProperties { + lineage: ColumnLineage, + sign: ValueSign, +} + +impl ColumnProperties { + /// A column the operator computes, with values of `sign`. + fn computed(sign: ValueSign) -> Self { + Self { + lineage: ColumnLineage::Computed, + sign, + } + } + + /// What two inputs' columns at one position agree on: each property + /// where they are equal, else `Computed` / `Unknown`. + fn agreed_with(self, other: &Self) -> Self { + Self { + lineage: if self.lineage == other.lineage { + self.lineage + } else { + ColumnLineage::Computed + }, + sign: agree(self.sign, other.sign), + } + } +} + +/// The lineage and sign of each column of `output`, the output schema of +/// `operator`, from its input groups' `column_lineage` and `column_signs` +/// and, for a `Scan`, its source and `source_facts`. +pub(super) fn column_lineage_and_signs( operator: &Operator>, output: &Schema, source_facts: &BTreeMap, -) -> Vec { - let signs = match operator { - Operator::NonASAP(op) => non_asap_signs(op, output, source_facts), - Operator::ASAP(op) => asap_signs(op, output), +) -> (Vec, Vec) { + let columns = match operator { + Operator::NonASAP(op) => non_asap_columns(op, output, source_facts), + Operator::ASAP(op) => asap_columns(op, output), }; debug_assert_eq!( - signs.len(), + columns.len(), output.fields.len(), - "one sign per output column of {}", + "one lineage and sign per output column of {}", operator.kind_name() ); - signs + columns + .into_iter() + .map(|column| (column.lineage, column.sign)) + .unzip() } -fn signs_of(input: &Input<'_>) -> Vec { - input.group_properties().column_signs.clone() +/// The columns of a memo group, read from its `group_properties`. +fn columns_of(input: &Input<'_>) -> Vec { + let properties = input.group_properties(); + properties + .column_lineage + .iter() + .zip(&properties.column_signs) + .map(|(lineage, &sign)| ColumnProperties { + lineage: lineage.clone(), + sign, + }) + .collect() } -fn non_asap_signs( +fn non_asap_columns( op: &NonASAPOp>, output: &Schema, source_facts: &BTreeMap, -) -> Vec { +) -> Vec { use NonASAPOp::*; - let unknown = || vec![Unknown; output.fields.len()]; + let unknown = || vec![ColumnProperties::computed(Unknown); output.fields.len()]; match op { - // The signs its source's facts declare (a histogram's or summary's - // series reads its family's); a counter's `value` is non-negative - // without a declaration. + // Each column copied from its source, with the sign its source's + // facts declare (a histogram's or summary's series reads its + // family's); a counter's `value` is non-negative without a + // declaration. Scan { source, schema, .. } => { let facts = series_facts(source_facts, source_name(source)); schema .fields .iter() - .map(|field| match facts { - Some((facts, series_type)) => facts - .column_signs - .get(&field.name) - .copied() - .or((field.name == "value" - && series_type == Some(PrometheusMetricType::Counter)) - .then_some(NonNegative)) - .unwrap_or(Unknown), - None => Unknown, + .map(|field| ColumnProperties { + sign: match facts { + Some((facts, series_type)) => facts + .column_signs + .get(&field.name) + .copied() + .or((field.name == "value" + && series_type == Some(PrometheusMetricType::Counter)) + .then_some(NonNegative)) + .unwrap_or(Unknown), + None => Unknown, + }, + lineage: ColumnLineage::SourceColumn(SourceColumn { + source: source.clone(), + column: field.name.clone(), + }), }) .collect() } Values { .. } => unknown(), // The scalar's value, every other column `Unknown`. PromqlVectorFromScalar(expr) => { - let mut signs = unknown(); + let mut columns = unknown(); if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, output) { - signs[value] = expression_sign(expr, &[]); + columns[value] = ColumnProperties::computed(expression_sign(expr, &[])); } - signs + columns } Filter { child, .. } | Sort { child, .. } @@ -116,77 +190,108 @@ fn non_asap_signs( | TimeShift { child, .. } | PromqlSubquery { child, .. } | PromqlSeriesSample { child, .. } - | PromqlInfoEnrich { child, .. } => signs_of(child), + | PromqlInfoEnrich { child, .. } => columns_of(child), // The relabeled column is written; every other column is copied. PromqlRelabel { dst, child, .. } => { - let mut signs = signs_of(child); + let mut columns = columns_of(child); + let written = ColumnProperties::computed(Unknown); match child.schema().column_id(dst) { - Some(written) => signs[written] = Unknown, - None => signs.push(Unknown), + Some(column) => columns[column] = written, + None => columns.push(written), } - signs + columns } - Project { cols, child, .. } => project_signs(cols, &signs_of(child)), + Project { cols, child, .. } => project_columns(cols, &columns_of(child)), Aggregate { reduction, measures, + output_names, child, .. } => { - if let [AggIntent::TopK { .. }] = measures.as_slice() { + if let [AggIntent::TopK { k, .. }] = measures.as_slice() { // The ranked rows, projected: each column copied. - return match crate::ir::schema::topk::ranked_row_projection(child, reduction, "") { - Ok(cols) => project_signs(&cols, &signs_of(child)), + let name = topk_output_name(*k, output_names); + return match ranked_row_projection(child, reduction, &name) { + Ok(cols) => project_columns(&cols, &columns_of(child)), Err(_) => unknown(), }; } - aggregate_signs(child, reduction, measures, output) + aggregate_columns(child, reduction, measures, output) } Join { kind, left, right, .. } => { - let mut signs = signs_of(left); + let mut columns = columns_of(left); if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { - signs.extend(signs_of(right)); + columns.extend(columns_of(right)); } - signs + columns } - SetOp { left, right, .. } => agreed(&[signs_of(left), signs_of(right)]), - Concat { children } => agreed(&children.iter().map(signs_of).collect::>()), + SetOp { left, right, .. } => agreed(&[columns_of(left), columns_of(right)]), + Concat { children } => agreed(&children.iter().map(columns_of).collect::>()), SQLWindowFunc { child, .. } => { - let mut signs = signs_of(child); - signs.push(Unknown); - signs + let mut columns = columns_of(child); + columns.push(ColumnProperties::computed(Unknown)); + columns } - // The left side's columns, its value per the operator, then the - // labels the match adds (unknown). + // The left side's columns, its value computed per the operator, + // then the right side's labels the match adds. A label that may + // come from either side (`or`, `group_right`) keeps what both sides + // agree on. BinaryOp { operator, lhs, rhs, .. } => { - let mut signs = signs_of(lhs); + let left = columns_of(lhs); + let right = columns_of(rhs); + let mut columns = left.clone(); + let right_named = |name: &str| { + rhs.schema() + .column_id(name) + .and_then(|column| right.get(column).cloned()) + .unwrap_or(ColumnProperties::computed(Unknown)) + }; + let right_rows = matches!(operator.kind, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)) + || operator + .vector_match + .as_ref() + .and_then(|m| m.grouping.as_ref()) + .is_some_and(|grouping| grouping.side == GroupSide::Right); + if right_rows { + for (column, field) in lhs.schema().fields.iter().enumerate() { + columns[column] = columns[column] + .clone() + .agreed_with(&right_named(&field.name)); + } + } if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { - let rhs_value = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) + let left_sign = left[value].sign; + let right_sign = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) .ok() - .map_or(Unknown, |column| signs_of(rhs)[column]); - signs[value] = match &operator.kind { + .and_then(|column| right.get(column)) + .map_or(Unknown, |column| column.sign); + columns[value] = ColumnProperties::computed(match &operator.kind { // A filter keeps the left side's values. BinaryOpKind::Compare(_) | BinaryOpKind::Set( PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless, - ) => signs[value], - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(signs[value], rhs_value), + ) => left_sign, + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(left_sign, right_sign), BinaryOpKind::CompareBool(_) => NonNegative, - BinaryOpKind::Arithmetic(op) => arithmetic_sign(op, signs[value], rhs_value), - }; + BinaryOpKind::Arithmetic(op) => arithmetic_sign(op, left_sign, right_sign), + }); } - signs.resize(output.fields.len(), Unknown); - signs + let added = output.fields.get(columns.len()..).unwrap_or_default(); + columns.extend(added.iter().map(|field| right_named(&field.name))); + columns.truncate(output.fields.len()); + columns } } } -fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { +fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec { use ASAPOp::*; + let unknown = || vec![ColumnProperties::computed(Unknown); output.fields.len()]; match op { SummaryAgg { child, @@ -194,8 +299,8 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { input, reduction, .. - } => state_signs(child, summary_type, input, reduction) - .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs), + } => state_columns(child, summary_type, input, reduction) + .map_or_else(unknown, |(_, columns)| columns), // The build's layout without its own time column, after the // computed `window_end` (column 0). SummaryTumblingWindowChain { @@ -204,15 +309,15 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { input, reduction, .. - } => match state_signs(child, summary_type, input, reduction) { - Some((build, mut signs)) => { + } => match state_columns(child, summary_type, input, reduction) { + Some((build, mut columns)) => { if let Some(time) = build.time_index { - signs.remove(time); + columns.remove(time); } - signs.insert(0, Unknown); - signs + columns.insert(0, ColumnProperties::computed(Unknown)); + columns } - None => vec![Unknown; output.fields.len()], + None => unknown(), }, SummaryMerge { children, group_by } => { // The windows a `TimeRange` over a chain selects merge into the @@ -226,11 +331,11 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { .. }) = summary_build(child) { - return state_signs(&child, &summary_type, &input, &reduction) - .map_or_else(|| vec![Unknown; output.fields.len()], |(_, signs)| signs); + return state_columns(&child, &summary_type, &input, &reduction) + .map_or_else(unknown, |(_, columns)| columns); } } - let merged = agreed(&children.iter().map(signs_of).collect::>()); + let merged = agreed(&children.iter().map(columns_of).collect::>()); match group_by { None => merged, Some(keys) => { @@ -241,7 +346,12 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { .position(|field| !field.is_plain()); keys.iter() .chain(state.as_ref()) - .map(|&column| merged.get(column).copied().unwrap_or(Unknown)) + .map(|&column| { + merged + .get(column) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)) + }) .collect() } } @@ -249,7 +359,7 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { SummaryEstimate { summary_input, query, - } => readout_signs(summary_input, output, |state, summary_type| match query { + } => readout_columns(summary_input, output, |state, summary_type| match query { SketchStatistic::Cardinality | SketchStatistic::FrequencyL2 | SketchStatistic::FrequencyEntropy => NonNegative, @@ -270,27 +380,37 @@ fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { FieldDataType::ExactAggregate(ExactKind::Count, _) ) }); - readout_signs( + readout_columns( child, output, |state, _| if counts { NonNegative } else { state }, ) } - MaintainPopulation { child, .. } => signs_of(child), - EvaluatePopulation { child, .. } if output == child.schema() => signs_of(child), + MaintainPopulation { child, .. } => columns_of(child), + EvaluatePopulation { child, .. } if output == child.schema() => columns_of(child), EvaluatePopulation { .. } | SummarySubtract { .. } | SummaryDelete { .. } | SummaryJoin { .. } - | Extension { .. } => vec![Unknown; output.fields.len()], + | Extension { .. } => unknown(), } } -/// A column item keeps its input column's sign; an expression item's sign -/// is [`expression_sign`]'s. -fn project_signs(cols: &[ProjectItem], input: &[ValueSign]) -> Vec { +/// A column item, renamed or not, copies its input column; an expression +/// item is computed, with [`expression_sign`]'s sign. +fn project_columns( + cols: &[ProjectItem], + input: &[ColumnProperties], +) -> Vec { + let signs: Vec = input.iter().map(|column| column.sign).collect(); cols.iter() - .map(|item| expression_sign(&item.expr, input)) + .map(|item| match item.expr { + ScalarExpr::Column(column) => input + .get(column) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)), + _ => ColumnProperties::computed(expression_sign(&item.expr, &signs)), + }) .collect() } @@ -322,46 +442,46 @@ fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { } /// An aggregate's, by where each output column comes from -/// ([`aggregate_output`]): a kept input column keeps its sign, a -/// measure's value has [`measure_sign`] of its input column's, and a label -/// a measure adds is `Unknown`. -fn aggregate_signs( +/// ([`aggregate_output`]): a kept input column is copied, a measure's +/// value is computed with [`measure_sign`] of its input column's sign, and +/// a label a measure adds is computed with no sign. +fn aggregate_columns( child: &Input<'_>, reduction: &Reduction, measures: &[AggIntent], output: &Schema, -) -> Vec { +) -> Vec { let Ok((_, sources)) = aggregate_output(child.schema(), reduction, measures, &[]) else { - return vec![Unknown; output.fields.len()]; + return vec![ColumnProperties::computed(Unknown); output.fields.len()]; }; - signs_from_sources(child, &sources, |index, input| { + columns_from_sources(child, &sources, |index, input| { measure_sign(&measures[index], input) }) } -/// The sign of each output column from `sources`: an input column keeps -/// its sign, measure `index` has `measure(index, the sign of its input -/// column)`, and an added label is `Unknown`. -fn signs_from_sources( +/// Each output column from `sources`: an input column is copied, measure +/// `index` is computed with `measure(index, the sign of its input +/// column)`, and an added label is computed with no sign. +fn columns_from_sources( child: &Input<'_>, sources: &[OutputColumnSource], measure: impl Fn(usize, ValueSign) -> ValueSign, -) -> Vec { - let input = &child.group_properties().column_signs; - let sign_of = |column: Option| { +) -> Vec { + let input = columns_of(child); + let column_at = |column: Option| { column - .and_then(|column| input.get(column).copied()) - .unwrap_or(Unknown) + .and_then(|column| input.get(column).cloned()) + .unwrap_or(ColumnProperties::computed(Unknown)) }; sources .iter() .map(|source| match source { - OutputColumnSource::InputColumn(column) => sign_of(Some(*column)), + OutputColumnSource::InputColumn(column) => column_at(Some(*column)), OutputColumnSource::Measure { index, input_column, - } => measure(*index, sign_of(*input_column)), - OutputColumnSource::AddedLabel => Unknown, + } => ColumnProperties::computed(measure(*index, column_at(*input_column).sign)), + OutputColumnSource::AddedLabel => ColumnProperties::computed(Unknown), }) .collect() } @@ -393,51 +513,50 @@ fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { } } -/// A summary state's schema and signs, in its layout -/// ([`summary_agg_layout`]): its key columns keep their input's sign and its -/// state column records the sign of the values it is updated with; `None` -/// when the layout does not derive. -fn state_signs( +/// A summary state's schema and columns, in its layout +/// ([`summary_agg_layout`]): its key columns copy their input's and its +/// state column is computed and records the sign of the values it is +/// updated with; `None` when the layout does not derive. +fn state_columns( child: &Input<'_>, summary_type: &FieldDataType, update: &SummaryUpdate, reduction: &Reduction, -) -> Option<(Schema, Vec)> { +) -> Option<(Schema, Vec)> { let (schema, sources) = summary_agg_layout(child.schema(), summary_type, reduction).ok()?; let updated_with = match &update.weight { SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) .ok() - .and_then(|column| signs_of(child).get(column).copied()) + .and_then(|column| child.group_properties().column_signs.get(column).copied()) .unwrap_or(Unknown), SummaryInputExpr::Constant(value) => sign_of_number(*value), SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, }; - let signs = signs_from_sources(child, &sources, |_, _| updated_with); - Some((schema, signs)) + let columns = columns_from_sources(child, &sources, |_, _| updated_with); + Some((schema, columns)) } -/// A readout of summary state: the state's plain columns keep their sign -/// in place, and the value read out has `value(the state's recorded sign, -/// the state column's type)`. -/// `Unknown` everywhere when the output does not keep the state's layout (a -/// top-k's rows). -fn readout_signs( +/// A readout of summary state: the state's plain columns are copied in +/// place, and the value read out is computed, with `value(the state's +/// recorded sign, the state column's type)`. All computed with no sign +/// when the output does not keep the state's layout (a top-k's rows). +fn readout_columns( state: &Input<'_>, output: &Schema, value: impl Fn(ValueSign, &FieldDataType) -> ValueSign, -) -> Vec { +) -> Vec { let input = state.schema(); if input.fields.len() != output.fields.len() { - return vec![Unknown; output.fields.len()]; + return vec![ColumnProperties::computed(Unknown); output.fields.len()]; } - signs_of(state) + columns_of(state) .into_iter() .zip(&input.fields) - .map(|(sign, field)| { + .map(|(column, field)| { if field.is_plain() { - sign + column } else { - value(sign, &field.dtype) + ColumnProperties::computed(value(column.sign, &field.dtype)) } }) .collect() @@ -470,17 +589,22 @@ fn quantile_sign(q: f64, input: ValueSign) -> ValueSign { } } -/// Per column, the inputs' sign if they all agree, else `Unknown`. -fn agreed(inputs: &[Vec]) -> Vec { +/// Per column, what all inputs agree on ([`ColumnProperties::agreed_with`]). +fn agreed(inputs: &[Vec]) -> Vec { let Some((first, rest)) = inputs.split_first() else { return Vec::new(); }; first .iter() .enumerate() - .map(|(column, &sign)| { - rest.iter().fold(sign, |agreed, other| { - agree(agreed, other.get(column).copied().unwrap_or(Unknown)) + .map(|(column, properties)| { + rest.iter().fold(properties.clone(), |agreed, other| { + agreed.agreed_with( + &other + .get(column) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)), + ) }) }) .collect() diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 474c4dd13..677ccaa0a 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -4,7 +4,7 @@ use std::collections::BTreeSet; use super::aggregate_coverage::{aggregate_coverage, AggregateCoverage, CoverageError}; -use super::column_properties::{column_signs, ValueSign}; +use super::column_properties::{column_lineage_and_signs, ColumnLineage, ValueSign}; use crate::ir::operator::arguments::{AtModifier, Source, TimeShift}; use crate::ir::operator::input::OperatorInput; use crate::ir::schema::Schema; @@ -60,8 +60,13 @@ impl MemoGroup { /// operator nodes. #[derive(Debug, Clone)] pub struct GroupProperties { + /// Fields: name, type, nullability; no table qualifier. pub schema: Schema, pub result_kind: OperatorResultKind, + /// Per output column, in schema order: the source column its values are + /// copied from, or `Computed` (concepts doc, "Column lineage"), from the + /// input groups' and, for a `Scan`, its source's columns. + pub column_lineage: Vec, /// Per output column, in schema order: the sign of its values, from the /// input groups' and, for a `Scan`, the declared ones. pub column_signs: Vec, @@ -128,8 +133,11 @@ pub(super) fn derive_group_properties( let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); let (schema, merge_coverage) = validate_expression_over_input_groups(&operator)?; let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(&operator).ok()); + let (column_lineage, column_signs) = + column_lineage_and_signs(&operator, &schema, &memo.source_facts); Ok(GroupProperties { - column_signs: column_signs(&operator, &schema, &memo.source_facts), + column_lineage, + column_signs, depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), diff --git a/crates/types/src/stages/logical/memo/tests.rs b/crates/types/src/stages/logical/memo/tests.rs index e64e092c5..69f81a7b1 100644 --- a/crates/types/src/stages/logical/memo/tests.rs +++ b/crates/types/src/stages/logical/memo/tests.rs @@ -32,7 +32,6 @@ fn project_first_column(input: MemoGroupId) -> Operator { alias: None, expr: crate::ir::ScalarExpr::Column(0), }], - qualifier: None, child: input, }) } @@ -562,7 +561,6 @@ fn literal(value: f64) -> impl FnOnce(&mut Memo) -> MemoGroupId { alias: Some("v".into()), expr: ScalarExpr::Literal(ScalarValue::Float64(value)), }], - qualifier: None, child, }) }) diff --git a/crates/types/src/stages/logical/mod.rs b/crates/types/src/stages/logical/mod.rs index 9e2b781cc..0658f80a5 100644 --- a/crates/types/src/stages/logical/mod.rs +++ b/crates/types/src/stages/logical/mod.rs @@ -16,7 +16,7 @@ pub use candidate::{ CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, LogicalProperties, LogicalPropertiesError, OriginOfCandidateForMemoGroup, }; -pub use column_properties::ValueSign; +pub use column_properties::{ColumnLineage, ValueSign}; pub use edge::LogicalEdge; pub use group::{GroupProperties, MemoGroup, MemoGroupError, MemoGroupId}; pub use memo::{Memo, MemoQuery}; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 3bfd01e38..8a6bdd5e2 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -14,7 +14,7 @@ mod pruning; pub use logical::{ view_coverage, AggregateCoverage, AggregateDefinition, CandidateError, CandidateExpression, - CandidateForMemoGroup, CandidateForMemoGroupId, CoverageError, GroupProperties, + CandidateForMemoGroup, CandidateForMemoGroupId, ColumnLineage, CoverageError, GroupProperties, LogicalASAPMemoCandidates, LogicalEdge, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroup, MemoGroupError, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, ValueSign, diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 60833bac8..d55290ee6 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -84,16 +84,10 @@ pub fn population_over_selector( ) -> Rc { use crate::ir::operator::arguments::Source; use crate::ir::operator::maintained_population::*; - if let Some(NonASAPOp::Project { - cols, - qualifier, - child, - }) = root.non_asap() - { + if let Some(NonASAPOp::Project { cols, child }) = root.non_asap() { let child = population_over_selector(child, statistic.clone(), grouping); let project = Operator::NonASAP(NonASAPOp::Project { cols: cols.clone(), - qualifier: qualifier.clone(), child, }); return Rc::new(OperatorNode::with_schema(project, root.schema.clone())); diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs new file mode 100644 index 000000000..6d57fdd5a --- /dev/null +++ b/crates/types/tests/column_lineage.rs @@ -0,0 +1,419 @@ +//! Column lineage, derived once per memo group from its input groups' +//! (concepts doc `memo-groups-properties-requirements.md`, section 3, +//! "Column lineage"): the doc's examples 1 to 3. Example 4 (PromQL label +//! matchers) is `scan_predicates_lift_and_their_states_merge` in +//! `memo_aggregate_coverage.rs`. +use asap_types::ir::operator::arguments::{AtModifier, JoinKind, Reduction, Source, SourceColumn}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::selection::Constraint; +use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, ScalarExpr, +}; +use asap_types::stages::{ + CandidateExpression, CandidateForMemoGroup, ColumnLineage, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::test_support::memo_fixtures::{filter, range, shift}; +use asap_types::workload::DataWorkload; + +fn memo() -> Memo { + Memo::new(&DataWorkload::default()) +} + +/// `Scan table` of Utf8 `columns`. +fn scan(table: &str, columns: &[&str]) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: table.into(), + }, + predicates: vec![], + schema: Schema::new( + columns + .iter() + .map(|name| Field::plain(*name, DataType::Utf8, false)) + .collect(), + ), + }) +} + +/// `column = value`. +fn equals(column: usize, value: &str) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(value.into()))), + semantics: ExprSemantics::Sql, + } +} + +/// `SummaryAgg[HLL](by keys; column)`: the distinct values of `column`. +fn hll(child: MemoGroupId, keys: Vec, column: &str) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Hll), + input: SummaryUpdate::column(ColumnRef::Named(column.into())), + reduction: Reduction::by(keys), + filter: None, + }) +} + +/// The column `column` of the table `table`. +fn column_of(table: &str, column: &str) -> SourceColumn { + SourceColumn { + source: Source::Table { + table_ref: table.into(), + }, + column: column.into(), + } +} + +fn source_column(table: &str, column: &str) -> ColumnLineage { + ColumnLineage::SourceColumn(column_of(table, column)) +} + +/// The column `column` of the series `latency`. +fn latency_column(column: &str) -> ColumnLineage { + ColumnLineage::SourceColumn(SourceColumn { + source: Source::TimeSeries { + metric: "latency".into(), + }, + column: column.into(), + }) +} + +fn lineage(memo: &Memo, group: MemoGroupId) -> &[ColumnLineage] { + &memo.groups()[group.0 as usize] + .group_properties + .column_lineage +} + +/// Example 1: `SELECT f.src, approx_distinct(f.dst) FROM flows f WHERE +/// f.proto = 'tcp' GROUP BY f.src`. The group-by key keeps `flows.src` +/// through the aggregate, the HLL state and its readout; the readout +/// renamed to the aggregate's column name has the aggregate's schema, so +/// `add_candidate` accepts it. +#[test] +fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { + let mut memo = memo(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst", "proto", "ts"])) + .unwrap(); + let g2 = memo + .find_or_add_group(filter(equals(2, "tcp"), g1)) + .unwrap(); + let g3 = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Cardinality { cols: vec![1] }], + output_names: vec!["approx_distinct(dst)".into()], + filters: vec![], + having: None, + child: g2, + })) + .unwrap(); + let g4 = memo.find_or_add_group(hll(g2, vec![0], "dst")).unwrap(); + let g5 = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Cardinality, + })) + .unwrap(); + let flows = ["src", "dst", "proto", "ts"].map(|column| source_column("flows", column)); + assert_eq!(lineage(&memo, g1), flows); + assert_eq!(lineage(&memo, g2), flows); + let key_then_computed = [source_column("flows", "src"), ColumnLineage::Computed]; + for group in [g3, g4, g5] { + assert_eq!(lineage(&memo, group), key_then_computed); + } + // The state's selection names the filtered column by its lineage. + let coverage = memo.groups()[g4.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert_eq!( + coverage.selection()[0].columns()[&column_of("flows", "proto")], + Constraint::In(vec![ScalarValue::Utf8("tcp".into())]) + ); + + let operator = Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: Some("src".into()), + expr: ScalarExpr::Column(0), + }, + ProjectItem { + alias: Some("approx_distinct(dst)".into()), + expr: ScalarExpr::Column(1), + }, + ], + child: g5, + }); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + let id = memo.add_candidate(g3, candidate).unwrap(); + assert_eq!(id.group, g3); +} + +/// Example 2: `SELECT source, count(*) FROM (SELECT f.src AS source FROM +/// flows f) GROUP BY source`. The renaming `Project` keeps `flows.src`; a +/// state grouped by `source` has the key lineage of one grouped by `src` +/// over the scan, a selection on the renamed `area` names `flows.region`, +/// and the rollup's group-key check, which compares lineage, passes. +#[test] +fn a_rename_keeps_lineage_and_the_rollup_key_check_compares_it() { + let mut memo = memo(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst", "region"])) + .unwrap(); + let renamed = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: [("source", 0), ("dst", 1), ("area", 2)] + .map(|(alias, column)| ProjectItem { + alias: Some(alias.into()), + expr: ScalarExpr::Column(column), + }) + .to_vec(), + child: g1, + })) + .unwrap(); + assert_eq!( + lineage(&memo, renamed), + [ + source_column("flows", "src"), + source_column("flows", "dst"), + source_column("flows", "region"), + ] + ); + + let by_source = memo + .find_or_add_group(hll(renamed, vec![0], "dst")) + .unwrap(); + let by_src = memo.find_or_add_group(hll(g1, vec![0], "dst")).unwrap(); + assert_eq!(lineage(&memo, by_source)[0], lineage(&memo, by_src)[0]); + + let in_us = memo + .find_or_add_group(filter(equals(2, "us"), renamed)) + .unwrap(); + let by_source_and_area = memo + .find_or_add_group(hll(in_us, vec![0, 2], "dst")) + .unwrap(); + let coverage = memo.groups()[by_source_and_area.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert_eq!( + coverage.selection()[0].columns()[&column_of("flows", "region")], + Constraint::In(vec![ScalarValue::Utf8("us".into())]) + ); + + let rolled_up = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![by_source_and_area], + group_by: Some(vec![0]), + })) + .unwrap(); + let rolled_up = &memo.groups()[rolled_up.0 as usize].group_properties; + assert_eq!( + rolled_up.column_lineage, + [source_column("flows", "src"), ColumnLineage::Computed] + ); + assert!(rolled_up.aggregate_coverage.is_some()); +} + +/// Example 3: `SELECT s.region, count(*) FROM sales s JOIN users u ON +/// s.user = u.id GROUP BY s.region`. Both join sides have a `region` +/// column; their lineage tells them apart, so selections on them are +/// different columns. +#[test] +fn join_columns_of_one_name_have_distinct_lineage() { + let mut memo = memo(); + let sales = memo + .find_or_add_group(scan("sales", &["user", "region"])) + .unwrap(); + let users = memo + .find_or_add_group(scan("users", &["id", "region"])) + .unwrap(); + let joined = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(2)), + semantics: ExprSemantics::Sql, + }), + left: sales, + right: users, + })) + .unwrap(); + assert_eq!( + lineage(&memo, joined), + [ + source_column("sales", "user"), + source_column("sales", "region"), + source_column("users", "id"), + source_column("users", "region"), + ] + ); + + let selected_column = |memo: &mut Memo, region_column: usize| { + let in_us = memo + .find_or_add_group(filter(equals(region_column, "us"), joined)) + .unwrap(); + let state = memo.find_or_add_group(hll(in_us, vec![], "id")).unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + selection.columns().keys().cloned().collect::>() + }; + assert_eq!( + selected_column(&mut memo, 1), + [column_of("sales", "region")] + ); + assert_eq!( + selected_column(&mut memo, 3), + [column_of("users", "region")] + ); +} + +/// A predicate on a computed column is not lifted: no selection can name +/// the column, so it stays in the definition's unlifted operators. +#[test] +fn a_predicate_on_a_computed_column_stays_unlifted() { + let mut memo = memo(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst"])) + .unwrap(); + let computed = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: Some("src".into()), + expr: ScalarExpr::Column(0), + }, + ProjectItem { + alias: Some("tag".into()), + expr: ScalarExpr::Literal(ScalarValue::Utf8("x".into())), + }, + ], + child: g1, + })) + .unwrap(); + assert_eq!( + lineage(&memo, computed), + [source_column("flows", "src"), ColumnLineage::Computed] + ); + let tagged = memo + .find_or_add_group(filter(equals(1, "x"), computed)) + .unwrap(); + let state = memo.find_or_add_group(hll(tagged, vec![], "src")).unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert!(coverage.selection()[0].columns().is_empty()); + assert!(coverage + .definition() + .unlifted_operators + .contains(&filter(equals(1, "x"), MemoGroupId(0)).map_children(|_| None))); +} + +/// `Scan latency`: `[ts, value, job]`, `ts` its time index. +fn latency_series() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, false), + ], + 0, + vec![], + ), + }) +} + +/// A chain of `SummaryAgg[KLL](by job; value)` windows: `[window_end, job, +/// state]`. `window_end` (column 0) and the state are computed and the key +/// keeps its lineage. A `TimeShift` and a `TimeRange` over the chain select +/// windows, so they pass the chain's lineage through; the merge of the +/// selected windows has the layout of the chain's build, `[job, state]`. +#[test] +fn a_chain_computes_window_end_and_state_and_its_window_selections_pass_lineage_through() { + let mut memo = memo(); + let series = memo.find_or_add_group(latency_series()).unwrap(); + let chain = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: series, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + tumbling_window_width_ms: 60_000, + })) + .unwrap(); + let chain_lineage = [ + ColumnLineage::Computed, + latency_column("job"), + ColumnLineage::Computed, + ]; + assert_eq!(lineage(&memo, chain), chain_lineage); + + let an_hour_ago = memo + .find_or_add_group(shift(3_600_000, None, chain)) + .unwrap(); + assert_eq!(lineage(&memo, an_hour_ago), chain_lineage); + let five_minutes_an_hour_ago = memo.find_or_add_group(range(300_000, an_hour_ago)).unwrap(); + assert_eq!(lineage(&memo, five_minutes_an_hour_ago), chain_lineage); + + let merged = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![five_minutes_an_hour_ago], + group_by: None, + })) + .unwrap(); + assert_eq!( + lineage(&memo, merged), + [latency_column("job"), ColumnLineage::Computed] + ); +} + +/// The PromQL front end lowers `m[5m] @ end()` as `TimeShift { at: End }` +/// above the selector lowered without `@`: it pins when the subtree is +/// read and copies its rows, so it passes the selector's lineage through, +/// as `@ start()` does. +#[test] +fn a_time_shift_at_end_or_start_above_a_selector_passes_its_lineage_through() { + let mut memo = memo(); + let series = memo.find_or_add_group(latency_series()).unwrap(); + let selector = memo.find_or_add_group(range(300_000, series)).unwrap(); + let series_lineage = ["ts", "value", "job"].map(latency_column); + for at in [AtModifier::End, AtModifier::Start] { + let pinned = memo + .find_or_add_group(shift(0, Some(at), selector)) + .unwrap(); + assert_eq!(lineage(&memo, pinned), series_lineage); + } +} diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index ad9e48481..ef8599935 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,7 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod column_lineage; mod memo_aggregate_coverage; mod memo_group_properties; mod physical_export; diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index 642134962..eebb90206 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -10,17 +10,17 @@ //! ``` use std::ops::Bound; -use asap_types::ir::operator::arguments::{AtModifier, Reduction}; +use asap_types::ir::operator::arguments::{AtModifier, Reduction, Source, SourceColumn}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::selection::{ColumnIdentity, Constraint, IntervalConstraintLimit}; +use asap_types::ir::properties::selection::{Constraint, IntervalConstraintLimit}; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr}; use asap_types::stages::{ - view_coverage, AggregateCoverage, CandidateExpression, CandidateForMemoGroup, CoverageError, - LogicalProperties, LogicalRequirements, Memo, MemoGroupError, MemoGroupId, + view_coverage, AggregateCoverage, CandidateExpression, CandidateForMemoGroup, ColumnLineage, + CoverageError, LogicalProperties, LogicalRequirements, Memo, MemoGroupError, MemoGroupId, OriginOfCandidateForMemoGroup, }; use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; @@ -145,8 +145,18 @@ fn last_minute() -> Constraint { relative(-MINUTE_MS, 0) } -fn ts() -> ColumnIdentity { - ColumnIdentity::new(None, "ts") +fn ts() -> SourceColumn { + latency_column("ts") +} + +/// The column `column` of the series `latency`. +fn latency_column(column: &str) -> SourceColumn { + SourceColumn { + source: Source::TimeSeries { + metric: "latency".into(), + }, + column: column.into(), + } } /// The constraint on the timestamp column of `group`'s one selection box. @@ -173,10 +183,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; - assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], - jobs(&["api"]) - ); + assert_eq!(selection.columns()[&latency_column("job")], jobs(&["api"])); assert_eq!(selection.columns()[&ts()], last_minute()); } @@ -199,7 +206,7 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { panic!("one box: {coverage:?}") }; assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], + selection.columns()[&latency_column("job")], jobs(&["api", "web"]) ); } @@ -243,7 +250,7 @@ fn a_filter_that_is_no_selection_stays_unlifted() { ] ); assert_eq!( - coverage_of_g10.selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage_of_g10.selection()[0].columns()[&latency_column("job")], jobs(&["api"]) ); assert!(matches!( @@ -296,19 +303,28 @@ fn states_whose_definitions_read_the_same_subquery_merge() { assert_eq!(definition, coverage(&memo, web).definition()); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, merged).selection()[0].columns()[&latency_column("job")], jobs(&["api", "web"]) ); } /// PromQL label matchers are the scan's predicates: they are lifted too, so /// `latency{job="api"}` and `latency{job="web"}` states merge. The scan, -/// without them, is the last unlifted operator. +/// without them, is the last unlifted operator. Each scan's `job` column has +/// lineage `latency.job`, so the selections `latency.job ∈ {api}` and +/// `latency.job ∈ {web}` are disjoint selections on one column (concepts doc, +/// section 3, column lineage example 4). #[test] fn scan_predicates_lift_and_their_states_merge() { let mut memo = Memo::new(&DataWorkload::default()); let state_of_job = |memo: &mut Memo, job: &str| { let series = memo.find_or_add_group(latency(vec![job_is(job)])).unwrap(); + assert_eq!( + memo.groups()[series.0 as usize] + .group_properties + .column_lineage[JOB], + ColumnLineage::SourceColumn(latency_column("job")) + ); let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); memo.find_or_add_group(kll(last_minute)).unwrap() }; @@ -321,12 +337,12 @@ fn scan_predicates_lift_and_their_states_merge() { vec![kll_unlifted(), latency(vec![]).map_children(|_| None)] ); assert_eq!( - coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, api).selection()[0].columns()[&latency_column("job")], jobs(&["api"]) ); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, merged).selection()[0].columns()[&latency_column("job")], jobs(&["api", "web"]) ); } @@ -490,7 +506,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { } ); assert_eq!( - coverage(&memo, g3).selection()[0].columns()[&ColumnIdentity::new(None, "value")], + coverage(&memo, g3).selection()[0].columns()[&latency_column("value")], Constraint::Interval { lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( 10.0 @@ -570,10 +586,7 @@ fn an_exact_aggregate_has_coverage() { let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; - assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], - jobs(&["api"]) - ); + assert_eq!(selection.columns()[&latency_column("job")], jobs(&["api"])); assert_eq!(selection.columns()[&ts()], last_minute()); } diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 005db69a0..f19b8c7b1 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -106,7 +106,6 @@ fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: @@ -308,7 +310,7 @@ G10.aggregate_coverage = AggregateCoverage { input: Some(G1), unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], }, - selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], + selection: [ { latency.job ∈ {api}, latency.ts ∈ (t − 60 s, t] } ], } ``` @@ -379,11 +381,11 @@ GROUP BY job; | Condition | Found at | Rule 1: same rows at the `SummaryAgg`? | Rule 2: simple? | Result | |---|---|---|---|---| -| `latency < 100` | `SummaryAgg.filter` | yes, it is already there | yes, an interval | `selection`: `latency ∈ (−∞, 100)` | -| `region = 'us'` | `Scan.predicates` | yes: `Project` passes `region` through (renamed `r`) | yes, a value set | `selection`: `r ∈ {us}` | +| `latency < 100` | `SummaryAgg.filter` | yes, it is already there | yes, an interval | `selection`: `t.latency ∈ (−∞, 100)` | +| `region = 'us'` | `Scan.predicates` | yes: `Project` passes `region` through (renamed `r`) | yes, a value set | `selection`: `t.region ∈ {us}` | | `latency * 2 > 10` | `Scan.predicates` | yes | **no**: it is on an expression, not a column | stays in `definition` | -So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL example with a time window is under **Time** below. +So the `selection` is `t.latency ∈ (−∞, 100)`, `t.region ∈ {us}`. A PromQL example with a time window is under **Time** below. **Rule 1 in detail.** Imagine moving the condition up, one operator at a time, until it is just below the `SummaryAgg`. Every operator it passes must leave the picked rows unchanged. Whether it can pass depends on what the operator does: @@ -391,16 +393,16 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | | `TimeRange` (range) or `TimeShift` (an offset or `@ T`; `@ start()` and `@ end()` are not lifted) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | -| `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | +| `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, on `r`, whose lineage is still `t.region`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | | any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit` | no | | | -A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. How a PromQL comparison is lowered decides whether it is such a `Filter`: +A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. But its condition is on the rate value, a computed column with no source column to name it by, so it stays in `definition` too. How a PromQL comparison is lowered decides whether it is such a `Filter`: | PromQL | Lowered to | Coverage | |---|---|---| -| `rate(m[5m]) > 0` (vector and scalar) | `Filter` | `selection`: `value ∈ (0, ∞)` | +| `rate(m[5m]) > 0` (vector and scalar) | `Filter` | stays in `definition`: `value` is computed by `rate` | | `rate(m[5m]) > bool 0` | `Project` (values become 0 or 1, no row is dropped) | stays in `definition`: it does not pick rows | | `rate(a[5m]) > rate(b[5m])` (vector and vector) | `BinaryOp` (matches the two sides by labels, like a join) | stays in `definition`: the walk does not enter a `BinaryOp` | @@ -426,11 +428,11 @@ Any other shape stays in `definition`: **Which column a condition is on.** -- A column is named by its table and name, `(table, name)`, **as the `SummaryAgg` reads it** (in the schema of its child). Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions. -- A `Project` gives its columns new names and replaces their table with its own qualifier (none by default): after `region AS r`, the column is `(none, r)`, so the condition becomes `r ∈ {us}`. Since the `definition` contains the same `Project`, two states that rename the same way still compare equal. -- Only a `Project` may rename. Any other operator whose output names a column differently from its input (allowed for every operator, e.g. a `Filter` with an explicit schema) stays in `definition`, and so do the conditions at and below it. Removing it would drop the rename: if a `Filter` calls the `tier` column `region`, its `tier = 'eu'` would read as `region ∈ {eu}` and look disjoint from another state's `region ∈ {us}`, though a row with `region = 'us', tier = 'eu'` is in both. -- If two columns the `SummaryAgg` reads have the same `(table, name)` (for example `Project [a AS k, b AS k]`), a condition on `k` cannot tell them apart and stays in `definition`. -- PromQL labels have no table, so a label is named by its name alone. +- A column is named by its **lineage** ([concepts doc, section 3](../concepts/memo-groups-properties-requirements.md#column-lineage)) in the group the `SummaryAgg` reads: the source column `(source, column)` its values are copied from, a `Scan`'s table or metric and its column there. Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions, though both columns are named `region`. +- A rename keeps lineage: after `Project [region AS r]`, `r` is still `t.region`, so the condition is `t.region ∈ {us}`. A memo group's schema is derived, so no other operator renames a column. Since the `definition` contains the same `Project`, two states that rename the same way still compare equal. +- A condition on a computed column (a `Project` expression item, an aggregate's measure, a readout's value) has no source column to name and stays in `definition`. +- If two columns the `SummaryAgg` reads have the same lineage (for example `Project [a AS k, a AS k2]`), the selection cannot name one of them apart, and their conditions stay in `definition`. +- A PromQL label is named by its metric and label: `latency.job`. - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. @@ -454,11 +456,11 @@ Any other shape stays in `definition`: | Query | Sub-DAG below `SummaryAgg` | `definition` keeps | `selection` takes | |---|---|---|---| -| SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `region ∈ {us}` | -| SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `latency ∈ (−∞, 100)` | -| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `ts ∈ (t − 3m, t − 2m]` | +| SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `t.region ∈ {us}` | +| SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `t.latency ∈ (−∞, 100)` | +| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `m.ts ∈ (t − 3m, t − 2m]` | | PromQL `quantile(0.99, rate(m{job="api"}[5m]))` | `rate(TimeRange(5m, Scan m {job = 'api'}))` | all of it | nothing: `job = 'api'` is under `rate`, which the walk cannot pass yet (Rule 1, later) | -| PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | `rate(TimeRange(5m, Scan m))` | `value ∈ (0, ∞)`: the `Filter` is above `rate` | +| PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | all of it | nothing: the `Filter` is above `rate`, but `value` is computed by `rate`, so no selection can name it | | PromQL `quantile(0.99, rate(a[5m]) > rate(b[5m]))` | `BinaryOp(>, rate(TimeRange(5m, Scan a)), rate(TimeRange(5m, Scan b)))` | all of it | nothing: a vector/vector comparison is a `BinaryOp`, not a `Filter` | `Scan t {…}` is a `Scan` with those predicates. In the last three rows `TimeRange(5m)` stays in `definition` in any case: it is the input window of `rate` and changes the rate values, so it does not just pick rows. @@ -556,7 +558,7 @@ With an error target, the planner answers the quantile from a KLL sketch with `k aggregate coverage of the SummaryAgg ┌──────────────────────────────────────────────────────────────────┐ │ definition: this SummaryAgg over Scan t (the Filter removed) │ -│ selection: region ∈ {us}, latency ∈ (−∞, 10000) │ +│ selection: t.region ∈ {us}, t.latency ∈ (−∞, 10000) │ └──────────────────────────────────────────────────────────────────┘ ``` @@ -795,13 +797,13 @@ The first two build `KLL(latency) by[job]` states. The third can be answered by │ State: job Utf8, state Sketch(KLL k=200) │ [[ SummaryMerge ]] - ▲ selection: region ∈ {us, eu} + ▲ selection: t.region ∈ {us, eu} │ ┌─────────────────┴─────────────────┐ │ │ [[ SummaryAgg ]] [[ SummaryAgg ]] KLL k=200, by job KLL k=200, by job - selection: region ∈ {us} selection: region ∈ {eu} + selection: t.region ∈ {us} selection: t.region ∈ {eu} ▲ ▲ │ │ [ Filter region = 'us' ] [ Filter region = 'eu' ] @@ -1124,9 +1126,6 @@ impl Schema { pub fn is_all_plain(&self) -> bool; /// Position of the first field with this name, or `None`. pub fn column_id(&self, name: &str) -> Option; - /// Position of the field with this table and name, or `None`. Tells apart - /// `a.k` and `b.k` after a join. - pub fn column_id_qualified(&self, table: &str, name: &str) -> Option; } /// One column of a `Schema`. `T` is `FieldDataType` on DAG edges, and plain @@ -1139,13 +1138,9 @@ pub struct Field { pub dtype: T, /// Whether the column may contain NULL. PromQL value columns are never NULL. pub nullable: bool, - /// Table or alias the column comes from (`t` in `t.col`), so two columns - /// with the same name from a join stay apart. `None` for PromQL labels and - /// unqualified columns. - pub table: Option, } impl Field { - /// An unqualified field holding a plain value of type `dtype`. + /// A field holding a plain value of type `dtype`. pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self; /// The value type if the field is plain; `None` for summary state. pub fn plain_dtype(&self) -> Option<&DataType>; @@ -1359,7 +1354,7 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/column_properties.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. @@ -1379,15 +1374,17 @@ pub struct SelectionBox { /// unrestricted. /// The timestamp column is one of them; there is no separate time /// field. - pub columns: BTreeMap, + pub columns: BTreeMap, } -/// Names a column across nodes, independent of its position. -pub struct ColumnIdentity { - /// The table or alias the column comes from; `None` if unqualified. - pub table: Option, - /// The column name. - pub name: String, +/// Names a column across nodes, independent of its position and name: the +/// source column of its lineage (`ColumnLineage::SourceColumn`), the same +/// type the lineage holds. +pub struct SourceColumn { + /// The table or metric a `Scan` reads. + pub source: Source, + /// The column's name in that source. + pub column: String, } /// A constraint on one column. From 152220ff6777018df11fb7f1151a58ee5b028c02 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:38:46 +0000 Subject: [PATCH 2/5] fix(types, frontends): lineage review fixes: computed identity, self-join scans, one-side match labels - ColumnLineage::Computed { group, column }: a computed column is named by the memo group and output column that computed it, so two computed columns are equal only when they are the same computation. The new group's id is passed to column_lineage_and_signs, derive_group_properties, validate_expression_over_input_groups and aggregate_coverage. A regrouping merge's state is computed by the merge. - SourceColumn gains scan_ordinal: a Scan is 0; at a Join and a vector-match BinaryOp the right side's columns of a source the left also scans shift past the left's ordinals, so a self-join's two sides are different keys. SourceColumn moves below impl Source and drops Hash; ColumnLineage drops Hash and serde. - project_lineage: one Project lineage rule for project_columns and lineage_at_top. - BinaryOp: labels named in group_left(...) take the right side's lineage and sign, in group_right(...) the left side's, overwriting a column of that name in place. - Front-end resolver: a qualified name never binds a column qualified by another table; Bound::new asserts one qualifier per column or none, and an aggregate whose measures add label columns is made unqualified at its call site. - Tests: self-join scan ordinals and alias-renamed key; three match-grouping label tests; qualified binding unit test. Design doc: scan_ordinal, Computed { group, column }, SourceColumn location, plain concepts-doc reference. --- crates/frontend-common/src/resolve.rs | 130 ++++++-- crates/types/src/ir/operator/arguments.rs | 25 +- crates/types/src/ir/properties/selection.rs | 2 +- .../src/stages/logical/aggregate_coverage.rs | 54 ++-- .../src/stages/logical/column_properties.rs | 225 +++++++++---- crates/types/src/stages/logical/group.rs | 24 +- crates/types/src/stages/logical/memo/mod.rs | 6 +- crates/types/tests/column_lineage.rs | 299 ++++++++++++++++-- crates/types/tests/memo_aggregate_coverage.rs | 1 + crates/types/tests/memo_group_properties.rs | 3 +- .../proposals/asap-primitive-schema.md | 20 +- 11 files changed, 615 insertions(+), 174 deletions(-) diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 76b6a7f88..77ac7ab14 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -62,23 +62,40 @@ impl<'a> Scope<'a> { } } - /// The column `column` names: by qualifier and name when it is - /// qualified and a column has both, else by its unique name. + /// The column `column` names: a qualified name by the column with that + /// qualifier and name, else by the unique column of that name whose + /// qualifier is not known (lost through a `Project` without alias or an + /// aggregate's measures); a column of that name qualified by another + /// table never matches. Any other name by its unique name. fn resolve(&self, column: &ColumnRef) -> Result { - if let ColumnRef::Qualified { table, name } = column { - let qualified = - self.schema - .fields - .iter() - .zip(self.qualifiers) - .position(|(field, qualifier)| { - field.name == *name && qualifier.as_deref() == Some(table.as_str()) - }); - if let Some(id) = qualified { - return Ok(id); - } + let ColumnRef::Qualified { table, name } = column else { + return resolve_column_ref(column, self.schema); + }; + let named = |wanted: Option<&str>| { + self.schema + .fields + .iter() + .enumerate() + .filter(|(id, field)| field.name == *name && self.qualifier_ref(*id) == wanted) + .map(|(id, _)| id) + .collect::>() + }; + let qualified = named(Some(table.as_str())); + let candidates = if qualified.is_empty() { + named(None) + } else { + qualified + }; + match candidates.as_slice() { + [id] => Ok(*id), + [] => Err(ResolveError::NotFound { + name: format!("{table}.{name}"), + available: self.schema.fields.iter().map(|f| f.name.clone()).collect(), + }), + _ => Err(ResolveError::Ambiguous { + name: format!("{table}.{name}"), + }), } - resolve_column_ref(column, self.schema) } fn resolve_all(&self, columns: &[ColumnRef]) -> Result, ResolveError> { @@ -87,7 +104,11 @@ impl<'a> Scope<'a> { /// The qualifier of column `column`. fn qualifier(&self, column: ColumnId) -> Option { - self.qualifiers.get(column).cloned().flatten() + self.qualifier_ref(column).map(str::to_owned) + } + + fn qualifier_ref(&self, column: ColumnId) -> Option<&str> { + self.qualifiers.get(column).and_then(Option::as_deref) } } @@ -99,13 +120,14 @@ struct Bound { } impl Bound { - /// `node` with `qualifiers`; unqualified unless there is one per column. + /// `node` with `qualifiers`: one per column, or none (unqualified). fn new(node: Rc, qualifiers: Vec>) -> Self { - let qualifiers = if qualifiers.len() == node.schema.fields.len() { - qualifiers - } else { - Vec::new() - }; + debug_assert!( + qualifiers.is_empty() || qualifiers.len() == node.schema.fields.len(), + "{} qualifiers for {} columns", + qualifiers.len(), + node.schema.fields.len() + ); Self { node, qualifiers } } @@ -369,6 +391,8 @@ fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result, _>>()?; // A `by` key keeps its qualifier; a per-series reduction keeps // its input's columns in place. A top-k emits ranked rows. + let out_schema = + aggregate_output_schema(&child.node.schema, &reduction, &measures, output_names)?; let qualifiers: Vec> = match &reduction { _ if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) => Vec::new(), Reduction::PerEntity => child.qualifiers.clone(), @@ -380,26 +404,23 @@ fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result ColumnRef { + ColumnRef::Qualified { + table: table.into(), + name: name.into(), + } + } + + /// A qualified name binds the column of that qualifier; a column of that + /// name qualified by another table never matches, so `x.src` over + /// `[u.src]` is an error, not `u.src`; a column whose qualifier is not + /// known matches by its unique name. + #[test] + fn a_qualified_name_never_binds_another_tables_column() { + let schema = Schema::new(vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("src", DataType::Utf8, false), + Field::plain("dst", DataType::Utf8, false), + ]); + let qualifiers = [Some("f".to_string()), Some("u".to_string()), None]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; + assert_eq!(scope.resolve(&qualified("u", "src")), Ok(1)); + assert!(matches!( + scope.resolve(&qualified("x", "src")), + Err(ResolveError::NotFound { .. }) + )); + assert_eq!(scope.resolve(&qualified("f", "dst")), Ok(2)); + // Before, a unique name matched whatever its qualifier. + let users = Schema::new(vec![Field::plain("src", DataType::Utf8, false)]); + let users_scope = Scope { + schema: &users, + qualifiers: &[Some("u".to_string())], + }; + assert!(matches!( + users_scope.resolve(&qualified("f", "src")), + Err(ResolveError::NotFound { .. }) + )); + } + fn eq_lit(col: UnresolvedScalar, v: &str) -> UnresolvedScalar { UnresolvedScalar::Compare { left: Box::new(col), diff --git a/crates/types/src/ir/operator/arguments.rs b/crates/types/src/ir/operator/arguments.rs index 57b7cd2c7..900458313 100644 --- a/crates/types/src/ir/operator/arguments.rs +++ b/crates/types/src/ir/operator/arguments.rs @@ -178,17 +178,6 @@ pub enum Source { Table { table_ref: String }, } -/// A column of a source (a `Scan`'s metric or table), by name: the lineage -/// of a column copied from it -/// (`ColumnLineage::SourceColumn`, a memo-group property in `crate::stages`), -/// and the key an aggregate coverage's selection names a column by. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SourceColumn { - pub source: Source, - pub column: String, -} - impl Source { /// Whether it is a time series (PromQL rows): a PromQL top-k over rows /// that scan only time series ranks one row per series. @@ -197,6 +186,20 @@ impl Source { } } +/// A column of one scan of a source (a `Scan`'s metric or table), by name: +/// the lineage of a column copied from it (`ColumnLineage::SourceColumn`, a +/// memo-group property in `crate::stages`), and the key an aggregate +/// coverage's selection names a column by. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceColumn { + pub source: Source, + /// Which scan of `source`: 0 for the first, counted left to right + /// across joins (a self-join's right side is 1). + pub scan_ordinal: u32, + pub column: String, +} + /// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's /// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison /// operator has exactly one representation (and one `Display`) across the IR. diff --git a/crates/types/src/ir/properties/selection.rs b/crates/types/src/ir/properties/selection.rs index 4ce95601b..06b7cc02c 100644 --- a/crates/types/src/ir/properties/selection.rs +++ b/crates/types/src/ir/properties/selection.rs @@ -13,7 +13,6 @@ use std::ops::Bound; use serde::{Deserialize, Serialize}; use crate::ir::operator::arguments::SourceColumn; - use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::ScalarExpr; use crate::ir::schema::ColumnId; @@ -576,6 +575,7 @@ mod tests { assert_eq!(last_5m.intersect(&at), None); let column = SourceColumn { source: Source::TimeSeries { metric: "m".into() }, + scan_ordinal: 0, column: "ts".into(), }; let one = |constraint: Constraint| SelectionBox { diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index a05b93f79..8719428e7 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -25,7 +25,7 @@ use std::ops::Bound; use thiserror::Error; -use super::column_properties::ColumnLineage; +use super::column_properties::{project_lineage, ColumnLineage}; use super::group::MemoGroupAsInput; use crate::ir::operator::arguments::{AtModifier, Reduction, SourceColumn}; use crate::ir::operator::asap::{selected_from, summary_agg_layout, ASAPOp}; @@ -160,7 +160,9 @@ fn stored_coverage<'m>(input: &MemoGroupAsInput<'m>) -> Option<&'m AggregateCove /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may /// also merge overlapping selections, and a counter family /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. +/// `group` is the memo group `operator` computes. pub(super) fn aggregate_coverage<'m>( + group: MemoGroupId, operator: &Operator>, ) -> Result { let as_built = |op: &ASAPOp>| -> ASAPOp> { @@ -170,7 +172,7 @@ pub(super) fn aggregate_coverage<'m>( Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { let top = unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); - of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) + of_aggregate(group, &InputOrTemporaryBuild::Input(*child), None, |_| top) } Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) if operator.output_kind() == OperatorResultKind::State => @@ -178,12 +180,12 @@ pub(super) fn aggregate_coverage<'m>( of_window_selection(operator) } Operator::NonASAP(_) => match selected_build(operator)? { - Some(build) => of_summary_agg(&build), + Some(build) => of_summary_agg(group, &build), None => Err(CoverageError::NoCoverage), }, Operator::ASAP( op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), - ) => of_summary_agg(&as_built(op)), + ) => of_summary_agg(group, &as_built(op)), Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { @@ -351,7 +353,8 @@ fn regrouped( let rows = build_input(first).ok_or(CoverageError::NoCoverage)?; let rows_lineage = &rows.group_properties().column_lineage; // The build's input column each key of the state names: the column of - // the key's lineage. + // the key's lineage (a computed key only the column of the same group + // and column). let mapped = keys .iter() .map(|&key| { @@ -424,8 +427,12 @@ fn time_column_of_build_input(state: &MemoGroupAsInput<'_>) -> Option>) -> Result { +/// what its `SummaryAgg` over the same input would. `group` is the memo +/// group being derived. +fn of_summary_agg( + group: MemoGroupId, + build: &ASAPOp>, +) -> Result { let (ASAPOp::SummaryAgg { child, summary_type: family, @@ -444,7 +451,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>) -> Result( + group: MemoGroupId, child: &Built<'m>, filter: Option<&Predicate>>, unlifted_top: impl FnOnce(Option>>) -> Operator>, @@ -491,7 +500,7 @@ fn of_aggregate<'m>( let InputOrTemporaryBuild::Input(base_input) = &base else { unreachable!("a rule builds only liftable operators over real inputs"); }; - let top = lineage_at_top(&base_input.group_properties().column_lineage, &chain); + let top = lineage_at_top(group, &base_input.group_properties().column_lineage, &chain); let top = top.as_slice(); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. @@ -657,24 +666,19 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option(base: &[ColumnLineage], chain: &[Operator]) -> Vec { +/// it: a `Filter`, `TimeRange` or `TimeShift` keeps its input's, and a +/// `Project`'s is [`project_lineage`]'s. The chain may hold a rule's +/// temporary build, which is no memo group, so a column computed in it is +/// `Computed` by `group`, the memo group being derived. +fn lineage_at_top( + group: MemoGroupId, + base: &[ColumnLineage], + chain: &[Operator], +) -> Vec { let mut lineage = base.to_vec(); for link in chain.iter().rev() { if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { - lineage = cols - .iter() - .map(|item| match item.expr { - ScalarExpr::Column(column) => lineage - .get(column) - .cloned() - .unwrap_or(ColumnLineage::Computed), - _ => ColumnLineage::Computed, - }) - .collect(); + lineage = project_lineage(group, cols, &lineage); } } lineage diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index e2d4efe16..4d381154a 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -4,15 +4,15 @@ //! its source's columns and the signs the data workload declares for them. //! One per-operator column layout says which input column each output //! column copies (lineage and sign kept) and which it computes (lineage -//! `Computed`, the sign by the operator's rule). Error contracts that hold -//! only for values of one sign read the sign; aggregate coverage names a -//! selection's columns by the lineage. +//! `Computed` by this group, the sign by the operator's rule). Error +//! contracts that hold only for values of one sign read the sign; +//! aggregate coverage names a selection's columns by the lineage. use std::collections::BTreeMap; -use super::group::MemoGroupAsInput; +use super::group::{MemoGroupAsInput, MemoGroupId}; use crate::ir::operator::arguments::{ - BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, SourceColumn, + BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, Source, SourceColumn, }; use crate::ir::operator::asap::{summary_agg_layout, summary_build}; use crate::ir::operator::input::OperatorInput; @@ -21,7 +21,7 @@ use crate::ir::scalar::expr_ir::{ArithmeticOpKind, ScalarValue}; use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; use crate::ir::schema::aggregate_schema::{aggregate_output, OutputColumnSource}; use crate::ir::schema::topk::{ranked_row_projection, topk_output_name}; -use crate::ir::schema::{FieldDataType, Schema}; +use crate::ir::schema::{ColumnId, FieldDataType, Schema}; use crate::ir::{ASAPOp, NonASAPOp, Operator}; use crate::summary_type::{ ExactKind, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, @@ -53,16 +53,25 @@ pub enum ValueSign { /// Where an output column's values come from, a group property derived /// from the input groups (concepts doc, section 3, "Column lineage"). It /// names a column across groups where a schema field's name cannot: two -/// join sides' columns of one name have different lineage, and a renamed -/// column keeps its lineage. Aggregate coverage names a selection's column -/// by it. -#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +/// join sides' columns of one name have different lineage, a self-join's +/// two scans of one source have different scan ordinals, and a renamed +/// column keeps its lineage. A column passed through unchanged keeps its +/// input column's lineage, so two columns have equal lineage only when +/// they hold the same values: the same column of the same scan, or the +/// same computation. Aggregate coverage names a selection's column by its +/// source column. +#[derive(Debug, Clone, PartialEq, Eq)] pub enum ColumnLineage { - /// Copied unchanged from a column of a source. + /// Copied unchanged from a column of one scan of a source. SourceColumn(SourceColumn), - /// Computed by an operator: a measure, an expression, a summary state or - /// a readout. - Computed, + /// Column `column` of memo group `group`, where it was computed: a + /// measure, an expression, a summary state or a readout, or a column + /// the inputs of a union-like operator (`SetOp`, `Concat`, + /// `SummaryMerge`) disagree on. + Computed { + group: MemoGroupId, + column: ColumnId, + }, } use ValueSign::{NonNegative, NonPositive, Unknown}; @@ -72,7 +81,9 @@ type Input<'m> = MemoGroupAsInput<'m>; /// One output column's lineage and sign. #[derive(Debug, Clone, PartialEq)] struct ColumnProperties { - lineage: ColumnLineage, + /// `None` for a column the group being derived computes: its lineage + /// is `Computed` by that group at the column's final position. + lineage: Option, sign: ValueSign, } @@ -80,35 +91,35 @@ impl ColumnProperties { /// A column the operator computes, with values of `sign`. fn computed(sign: ValueSign) -> Self { Self { - lineage: ColumnLineage::Computed, + lineage: None, sign, } } /// What two inputs' columns at one position agree on: each property - /// where they are equal, else `Computed` / `Unknown`. + /// where they are equal, else computed by this group / `Unknown`. fn agreed_with(self, other: &Self) -> Self { Self { - lineage: if self.lineage == other.lineage { - self.lineage - } else { - ColumnLineage::Computed - }, + lineage: self + .lineage + .filter(|lineage| other.lineage.as_ref() == Some(lineage)), sign: agree(self.sign, other.sign), } } } /// The lineage and sign of each column of `output`, the output schema of -/// `operator`, from its input groups' `column_lineage` and `column_signs` -/// and, for a `Scan`, its source and `source_facts`. +/// `operator` computed by memo group `group`, from its input groups' +/// `column_lineage` and `column_signs` and, for a `Scan`, its source and +/// `source_facts`. pub(super) fn column_lineage_and_signs( + group: MemoGroupId, operator: &Operator>, output: &Schema, source_facts: &BTreeMap, ) -> (Vec, Vec) { let columns = match operator { - Operator::NonASAP(op) => non_asap_columns(op, output, source_facts), + Operator::NonASAP(op) => non_asap_columns(group, op, output, source_facts), Operator::ASAP(op) => asap_columns(op, output), }; debug_assert_eq!( @@ -119,7 +130,14 @@ pub(super) fn column_lineage_and_signs( ); columns .into_iter() - .map(|column| (column.lineage, column.sign)) + .enumerate() + .map(|(index, properties)| { + let lineage = properties.lineage.unwrap_or(ColumnLineage::Computed { + group, + column: index, + }); + (lineage, properties.sign) + }) .unzip() } @@ -131,13 +149,40 @@ fn columns_of(input: &Input<'_>) -> Vec { .iter() .zip(&properties.column_signs) .map(|(lineage, &sign)| ColumnProperties { - lineage: lineage.clone(), + lineage: Some(lineage.clone()), sign, }) .collect() } +/// `right`, the right side's columns of an operator that concatenates two +/// inputs' columns after `left`'s: a source column of a source `left` also +/// scans moves to the scans after `left`'s (its scan ordinal shifted by +/// `left`'s largest for that source, plus one), so a self-join's two sides +/// name different scans. +fn after_left_scans( + left: &[ColumnProperties], + mut right: Vec, +) -> Vec { + let mut first_free_scan: BTreeMap<&Source, u32> = BTreeMap::new(); + for column in left { + if let Some(ColumnLineage::SourceColumn(source_column)) = &column.lineage { + let free = first_free_scan.entry(&source_column.source).or_default(); + *free = (*free).max(source_column.scan_ordinal + 1); + } + } + for column in &mut right { + if let Some(ColumnLineage::SourceColumn(source_column)) = &mut column.lineage { + if let Some(shift) = first_free_scan.get(&source_column.source) { + source_column.scan_ordinal += shift; + } + } + } + right +} + fn non_asap_columns( + group: MemoGroupId, op: &NonASAPOp>, output: &Schema, source_facts: &BTreeMap, @@ -166,10 +211,11 @@ fn non_asap_columns( .unwrap_or(Unknown), None => Unknown, }, - lineage: ColumnLineage::SourceColumn(SourceColumn { + lineage: Some(ColumnLineage::SourceColumn(SourceColumn { source: source.clone(), + scan_ordinal: 0, column: field.name.clone(), - }), + })), }) .collect() } @@ -201,7 +247,7 @@ fn non_asap_columns( } columns } - Project { cols, child, .. } => project_columns(cols, &columns_of(child)), + Project { cols, child, .. } => project_columns(group, cols, child), Aggregate { reduction, measures, @@ -213,7 +259,7 @@ fn non_asap_columns( // The ranked rows, projected: each column copied. let name = topk_output_name(*k, output_names); return match ranked_row_projection(child, reduction, &name) { - Ok(cols) => project_columns(&cols, &columns_of(child)), + Ok(cols) => project_columns(group, &cols, child), Err(_) => unknown(), }; } @@ -224,7 +270,8 @@ fn non_asap_columns( } => { let mut columns = columns_of(left); if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { - columns.extend(columns_of(right)); + let right = after_left_scans(&columns, columns_of(right)); + columns.extend(right); } columns } @@ -238,30 +285,40 @@ fn non_asap_columns( // The left side's columns, its value computed per the operator, // then the right side's labels the match adds. A label that may // come from either side (`or`, `group_right`) keeps what both sides - // agree on. + // agree on; a label `group_left(...)` names comes from the right + // ("one") side and one `group_right(...)` names from the left, + // overwriting a column of that name. A vector match is a join, so + // the right side reads scans after the left's; `or` is a union. BinaryOp { operator, lhs, rhs, .. } => { + let is_or = matches!(operator.kind, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)); let left = columns_of(lhs); - let right = columns_of(rhs); - let mut columns = left.clone(); - let right_named = |name: &str| { - rhs.schema() + let right = if is_or { + columns_of(rhs) + } else { + after_left_scans(&left, columns_of(rhs)) + }; + let named = |input: &Input<'_>, columns: &[ColumnProperties], name: &str| { + input + .schema() .column_id(name) - .and_then(|column| right.get(column).cloned()) + .and_then(|column| columns.get(column).cloned()) .unwrap_or(ColumnProperties::computed(Unknown)) }; - let right_rows = matches!(operator.kind, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)) - || operator - .vector_match - .as_ref() - .and_then(|m| m.grouping.as_ref()) - .is_some_and(|grouping| grouping.side == GroupSide::Right); + let grouping = operator + .vector_match + .as_ref() + .and_then(|m| m.grouping.as_ref()); + let right_rows = + is_or || grouping.is_some_and(|grouping| grouping.side == GroupSide::Right); + let mut columns = left.clone(); if right_rows { for (column, field) in lhs.schema().fields.iter().enumerate() { - columns[column] = columns[column] - .clone() - .agreed_with(&right_named(&field.name)); + columns[column] = + columns[column] + .clone() + .agreed_with(&named(rhs, &right, &field.name)); } } if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { @@ -282,8 +339,19 @@ fn non_asap_columns( }); } let added = output.fields.get(columns.len()..).unwrap_or_default(); - columns.extend(added.iter().map(|field| right_named(&field.name))); + columns.extend(added.iter().map(|field| named(rhs, &right, &field.name))); columns.truncate(output.fields.len()); + if let Some(grouping) = grouping { + let (one_side, one_side_columns) = match grouping.side { + GroupSide::Left => (rhs, &right), + GroupSide::Right => (lhs, &left), + }; + for label in &grouping.labels { + if let Some(column) = output.column_id(label) { + columns[column] = named(one_side, one_side_columns, label); + } + } + } columns } } @@ -338,20 +406,24 @@ fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec>()); match group_by { None => merged, + // The keys copied; the state, merged across the inputs' + // groups, computed with the sign they agree on. Some(keys) => { + let column_at = |column: usize| { + merged + .get(column) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)) + }; let state = children[0] .schema() .fields .iter() - .position(|field| !field.is_plain()); + .position(|field| !field.is_plain()) + .map(|state| ColumnProperties::computed(column_at(state).sign)); keys.iter() - .chain(state.as_ref()) - .map(|&column| { - merged - .get(column) - .cloned() - .unwrap_or(ColumnProperties::computed(Unknown)) - }) + .map(|&key| column_at(key)) + .chain(state) .collect() } } @@ -396,20 +468,41 @@ fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec( + group: MemoGroupId, cols: &[ProjectItem], - input: &[ColumnProperties], + child: &Input<'_>, ) -> Vec { - let signs: Vec = input.iter().map(|column| column.sign).collect(); + let input = child.group_properties(); + project_lineage(group, cols, &input.column_lineage) + .into_iter() + .zip(cols) + .map(|(lineage, item)| ColumnProperties { + lineage: Some(lineage), + sign: expression_sign(&item.expr, &input.column_signs), + }) + .collect() +} + +/// The lineage of the output columns of `cols`, a `Project` computed by +/// memo group `group` over input columns of lineage `input`: a column +/// item, renamed or not, copies its input column's; an expression item is +/// `Computed` by `group` at its position. +pub(super) fn project_lineage( + group: MemoGroupId, + cols: &[ProjectItem], + input: &[ColumnLineage], +) -> Vec { cols.iter() - .map(|item| match item.expr { - ScalarExpr::Column(column) => input - .get(column) - .cloned() - .unwrap_or(ColumnProperties::computed(Unknown)), - _ => ColumnProperties::computed(expression_sign(&item.expr, &signs)), + .enumerate() + .map(|(column, item)| { + match item.expr { + ScalarExpr::Column(input_column) => input.get(input_column).cloned(), + _ => None, + } + .unwrap_or(ColumnLineage::Computed { group, column }) }) .collect() } diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 677ccaa0a..78992caa8 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -60,12 +60,13 @@ impl MemoGroup { /// operator nodes. #[derive(Debug, Clone)] pub struct GroupProperties { - /// Fields: name, type, nullability; no table qualifier. + /// Fields: name, type, nullability. pub schema: Schema, pub result_kind: OperatorResultKind, - /// Per output column, in schema order: the source column its values are - /// copied from, or `Computed` (concepts doc, "Column lineage"), from the - /// input groups' and, for a `Scan`, its source's columns. + /// Per output column, in schema order: the source column (of one scan) + /// its values are copied from, or the group and column where they were + /// computed (concepts doc, section 3, "Column lineage"), from the input + /// groups' and, for a `Scan`, its source's columns. pub column_lineage: Vec, /// Per output column, in schema order: the sign of its values, from the /// input groups' and, for a `Scan`, the declared ones. @@ -107,34 +108,37 @@ pub enum MemoGroupError { /// Whether `operator` is a valid expression over its input groups /// ([`MemoGroupAsInput`]), never a node tree: its output schema derives, its /// inputs pass validation, and a merge reads only states it may merge. +/// `group` is the memo group the expression computes (new or existing). /// Returns the output schema and, for a merge, the coverage its check /// derived. pub(super) fn validate_expression_over_input_groups( + group: MemoGroupId, operator: &Operator>, ) -> Result<(Schema, Option), MemoGroupError> { let schema = operator.output_schema()?; operator.validate_inputs()?; // A merge is valid only over states of one definition it may merge. let merge_coverage = match operator { - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(aggregate_coverage(operator)?), + Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(aggregate_coverage(group, operator)?), _ => None, }; Ok((schema, merge_coverage)) } -/// The properties of a group computing `expression` over its input -/// groups, once it is valid over them +/// The properties of new memo group `group` computing `expression` over +/// its input groups, once it is valid over them /// ([`validate_expression_over_input_groups`]), reading each input group's /// `group_properties` and `expression`. pub(super) fn derive_group_properties( memo: &Memo, + group: MemoGroupId, expression: &Operator, ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); - let (schema, merge_coverage) = validate_expression_over_input_groups(&operator)?; - let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(&operator).ok()); + let (schema, merge_coverage) = validate_expression_over_input_groups(group, &operator)?; + let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(group, &operator).ok()); let (column_lineage, column_signs) = - column_lineage_and_signs(&operator, &schema, &memo.source_facts); + column_lineage_and_signs(group, &operator, &schema, &memo.source_facts); Ok(GroupProperties { column_lineage, column_signs, diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 1bbafc4ec..957897109 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -124,11 +124,11 @@ impl Memo { { return Ok(*group); } - let group_properties = derive_group_properties(self, &expression)?; - let shareable = group_properties.shareable_when_equal; let id = MemoGroupId( u32::try_from(self.groups.len()).expect("a memo has at most u32::MAX groups"), ); + let group_properties = derive_group_properties(self, id, &expression)?; + let shareable = group_properties.shareable_when_equal; for &input in expression.children() { add_parent(&mut self.groups[input.0 as usize], id); } @@ -182,7 +182,7 @@ impl Memo { memo: self, group_id, }); - let (schema, _) = validate_expression_over_input_groups(&operator) + let (schema, _) = validate_expression_over_input_groups(group, &operator) .map_err(CandidateError::NotValidOverInputGroups)?; let result_kind = operator.output_kind(); let group_properties = &self.groups[group.0 as usize].group_properties; diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs index 6d57fdd5a..dc324825a 100644 --- a/crates/types/tests/column_lineage.rs +++ b/crates/types/tests/column_lineage.rs @@ -2,23 +2,29 @@ //! (concepts doc `memo-groups-properties-requirements.md`, section 3, //! "Column lineage"): the doc's examples 1 to 3. Example 4 (PromQL label //! matchers) is `scan_predicates_lift_and_their_states_merge` in -//! `memo_aggregate_coverage.rs`. -use asap_types::ir::operator::arguments::{AtModifier, JoinKind, Reduction, Source, SourceColumn}; +//! `memo_aggregate_coverage.rs`. Then a self-join's two scans, and the +//! lineage and sign of PromQL match-grouping labels. +use std::collections::BTreeMap; + +use asap_types::ir::operator::arguments::{ + AtModifier, BinaryOpKind, GroupSide, JoinKind, Reduction, Source, SourceColumn, VectorGrouping, + VectorMatch, VectorMatchKind, +}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::selection::Constraint; -use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, ScalarExpr, + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, ScalarExpr, }; use asap_types::stages::{ CandidateExpression, CandidateForMemoGroup, ColumnLineage, LogicalProperties, - LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, ValueSign, }; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; use asap_types::test_support::memo_fixtures::{filter, range, shift}; -use asap_types::workload::DataWorkload; +use asap_types::workload::{DataWorkload, SourceFacts}; fn memo() -> Memo { Memo::new(&DataWorkload::default()) @@ -61,12 +67,18 @@ fn hll(child: MemoGroupId, keys: Vec, column: &str) -> Operator SourceColumn { + column_of_scan(table, 0, column) +} + +/// The column `column` of scan `scan_ordinal` of the table `table`. +fn column_of_scan(table: &str, scan_ordinal: u32, column: &str) -> SourceColumn { SourceColumn { source: Source::Table { table_ref: table.into(), }, + scan_ordinal, column: column.into(), } } @@ -81,10 +93,16 @@ fn latency_column(column: &str) -> ColumnLineage { source: Source::TimeSeries { metric: "latency".into(), }, + scan_ordinal: 0, column: column.into(), }) } +/// Column `column` of memo group `group`, where it was computed. +fn computed(group: MemoGroupId, column: usize) -> ColumnLineage { + ColumnLineage::Computed { group, column } +} + fn lineage(memo: &Memo, group: MemoGroupId) -> &[ColumnLineage] { &memo.groups()[group.0 as usize] .group_properties @@ -125,9 +143,13 @@ fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { let flows = ["src", "dst", "proto", "ts"].map(|column| source_column("flows", column)); assert_eq!(lineage(&memo, g1), flows); assert_eq!(lineage(&memo, g2), flows); - let key_then_computed = [source_column("flows", "src"), ColumnLineage::Computed]; + // Each of the aggregate, the state and the readout computes its own + // second column. for group in [g3, g4, g5] { - assert_eq!(lineage(&memo, group), key_then_computed); + assert_eq!( + lineage(&memo, group), + [source_column("flows", "src"), computed(group, 1)] + ); } // The state's selection names the filtered column by its lineage. let coverage = memo.groups()[g4.0 as usize] @@ -218,16 +240,17 @@ fn a_rename_keeps_lineage_and_the_rollup_key_check_compares_it() { Constraint::In(vec![ScalarValue::Utf8("us".into())]) ); - let rolled_up = memo + let rolled_up_id = memo .find_or_add_group(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![by_source_and_area], group_by: Some(vec![0]), })) .unwrap(); - let rolled_up = &memo.groups()[rolled_up.0 as usize].group_properties; + let rolled_up = &memo.groups()[rolled_up_id.0 as usize].group_properties; + // The regrouped state is computed by the merge, not its input's. assert_eq!( rolled_up.column_lineage, - [source_column("flows", "src"), ColumnLineage::Computed] + [source_column("flows", "src"), computed(rolled_up_id, 1)] ); assert!(rolled_up.aggregate_coverage.is_some()); } @@ -301,7 +324,7 @@ fn a_predicate_on_a_computed_column_stays_unlifted() { let g1 = memo .find_or_add_group(scan("flows", &["src", "dst"])) .unwrap(); - let computed = memo + let tagged_rows = memo .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { cols: vec![ ProjectItem { @@ -317,11 +340,11 @@ fn a_predicate_on_a_computed_column_stays_unlifted() { })) .unwrap(); assert_eq!( - lineage(&memo, computed), - [source_column("flows", "src"), ColumnLineage::Computed] + lineage(&memo, tagged_rows), + [source_column("flows", "src"), computed(tagged_rows, 1)] ); let tagged = memo - .find_or_add_group(filter(equals(1, "x"), computed)) + .find_or_add_group(filter(equals(1, "x"), tagged_rows)) .unwrap(); let state = memo.find_or_add_group(hll(tagged, vec![], "src")).unwrap(); let coverage = memo.groups()[state.0 as usize] @@ -375,9 +398,9 @@ fn a_chain_computes_window_end_and_state_and_its_window_selections_pass_lineage_ })) .unwrap(); let chain_lineage = [ - ColumnLineage::Computed, + computed(chain, 0), latency_column("job"), - ColumnLineage::Computed, + computed(chain, 2), ]; assert_eq!(lineage(&memo, chain), chain_lineage); @@ -396,7 +419,7 @@ fn a_chain_computes_window_end_and_state_and_its_window_selections_pass_lineage_ .unwrap(); assert_eq!( lineage(&memo, merged), - [latency_column("job"), ColumnLineage::Computed] + [latency_column("job"), computed(merged, 1)] ); } @@ -417,3 +440,241 @@ fn a_time_shift_at_end_or_start_above_a_selector_passes_its_lineage_through() { assert_eq!(lineage(&memo, pinned), series_lineage); } } + +/// `SELECT approx_distinct(

_dst) FROM (SELECT a.src AS

_src, a.proto +/// AS

_proto, b.dst AS

_dst, b.proto AS

_other_proto FROM flows a +/// JOIN flows b ON a.src = b.src WHERE .proto = 'tcp')`, aliases +/// prefixed by `prefix`, filtering the left side's `proto` (column 2 of the +/// join) or the right side's (column 5): the HLL state's selection keys. +fn self_join_selection(memo: &mut Memo, prefix: &str, proto_column: usize) -> Vec { + let flows = memo + .find_or_add_group(scan("flows", &["src", "dst", "proto"])) + .unwrap(); + let joined = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(3)), + semantics: ExprSemantics::Sql, + }), + left: flows, + right: flows, + })) + .unwrap(); + let tcp = memo + .find_or_add_group(filter(equals(proto_column, "tcp"), joined)) + .unwrap(); + let renamed = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: [("src", 0), ("proto", 2), ("dst", 4), ("other_proto", 5)] + .map(|(alias, column)| ProjectItem { + alias: Some(format!("{prefix}_{alias}")), + expr: ScalarExpr::Column(column), + }) + .to_vec(), + child: tcp, + })) + .unwrap(); + let state = memo + .find_or_add_group(hll(renamed, vec![], &format!("{prefix}_dst"))) + .unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + selection.columns().keys().cloned().collect() +} + +/// A self-join's two scans of one source have different scan ordinals: the +/// left side's columns are scan 0's, the right side's scan 1's. So a +/// predicate on the left side's `proto` lifts keyed by `flows` scan 0 +/// `proto`, one on the right side's by scan 1, and renaming the columns +/// (other aliases) keeps the key. +#[test] +fn a_self_join_names_its_two_scans_apart() { + let mut memo = memo(); + let flows = memo + .find_or_add_group(scan("flows", &["src", "dst", "proto"])) + .unwrap(); + let joined = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(3)), + semantics: ExprSemantics::Sql, + }), + left: flows, + right: flows, + })) + .unwrap(); + assert_eq!( + lineage(&memo, joined), + [ + (0, "src"), + (0, "dst"), + (0, "proto"), + (1, "src"), + (1, "dst"), + (1, "proto") + ] + .map(|(scan, column)| ColumnLineage::SourceColumn(column_of_scan("flows", scan, column))) + ); + + let left_proto = [column_of_scan("flows", 0, "proto")]; + assert_eq!(self_join_selection(&mut memo, "a", 2), left_proto); + assert_eq!(self_join_selection(&mut memo, "x", 2), left_proto); + assert_eq!( + self_join_selection(&mut memo, "a", 5), + [column_of_scan("flows", 1, "proto")] + ); +} + +/// The memo of the match-grouping examples: `team` declared non-negative +/// in `a` and non-positive in `b`. +fn match_grouping_memo() -> Memo { + let declared = |sign| SourceFacts { + column_signs: BTreeMap::from([("team".to_string(), sign)]), + ..SourceFacts::default() + }; + Memo::new(&DataWorkload { + sources: BTreeMap::from([ + ("a".to_string(), declared(ValueSign::NonNegative)), + ("b".to_string(), declared(ValueSign::NonPositive)), + ]), + ..DataWorkload::default() + }) +} + +/// `Scan metric`: `[ts, value, *labels]`, `ts` its time index. +fn series(metric: &str, labels: &[&str]) -> Operator { + let mut fields = vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]; + fields.extend( + labels + .iter() + .map(|label| Field::plain(*label, DataType::Utf8, false)), + ); + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index(fields, 0, vec![]), + }) +} + +/// `lhs * on(job) group_(team) rhs`. +fn grouped_product(side: GroupSide, lhs: MemoGroupId, rhs: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + vector_match: Some(VectorMatch { + kind: VectorMatchKind::On, + labels: vec!["job".into()], + grouping: Some(VectorGrouping { + side, + labels: vec!["team".into()], + }), + }), + }, + return_bool: false, + lhs, + rhs, + }) +} + +/// The lineage and sign of column `name` of `group`. +fn lineage_and_sign_of(memo: &Memo, group: MemoGroupId, name: &str) -> (ColumnLineage, ValueSign) { + let properties = &memo.groups()[group.0 as usize].group_properties; + let column = properties + .schema + .column_id(name) + .expect("a column of that name"); + ( + properties.column_lineage[column].clone(), + properties.column_signs[column], + ) +} + +fn series_column(metric: &str, column: &str) -> ColumnLineage { + ColumnLineage::SourceColumn(SourceColumn { + source: Source::TimeSeries { + metric: metric.into(), + }, + scan_ordinal: 0, + column: column.into(), + }) +} + +/// `a * on(job) group_left(team) b`: `team`, which only `b` has, takes +/// `b.team`'s lineage and sign (the "one" side's). +#[test] +fn a_group_left_label_comes_from_the_right_side() { + let mut memo = match_grouping_memo(); + let a = memo.find_or_add_group(series("a", &["job"])).unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Left, a, b)) + .unwrap(); + assert_eq!( + lineage_and_sign_of(&memo, product, "team"), + (series_column("b", "team"), ValueSign::NonPositive) + ); +} + +/// `a * on(job) group_left(team) b` with `a` having its own `team`: PromQL +/// overwrites it with `b`'s, so the column takes `b.team`'s lineage and +/// sign in place. +#[test] +fn a_group_left_label_overwrites_the_left_sides_column_of_that_name() { + let mut memo = match_grouping_memo(); + let a = memo + .find_or_add_group(series("a", &["job", "team"])) + .unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Left, a, b)) + .unwrap(); + let schema = &memo.groups()[product.0 as usize].group_properties.schema; + assert_eq!(schema.fields.len(), 4, "no second `team` column"); + assert_eq!( + lineage_and_sign_of(&memo, product, "team"), + (series_column("b", "team"), ValueSign::NonPositive) + ); +} + +/// `a * on(job) group_right(team) b`: `team` takes `a.team`'s lineage and +/// sign (the left, "one" side's). +#[test] +fn a_group_right_label_comes_from_the_left_side() { + let mut memo = match_grouping_memo(); + let a = memo + .find_or_add_group(series("a", &["job", "team"])) + .unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Right, a, b)) + .unwrap(); + assert_eq!( + lineage_and_sign_of(&memo, product, "team"), + (series_column("a", "team"), ValueSign::NonNegative) + ); +} diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index eebb90206..b5a80d3fb 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -155,6 +155,7 @@ fn latency_column(column: &str) -> SourceColumn { source: Source::TimeSeries { metric: "latency".into(), }, + scan_ordinal: 0, column: column.into(), } } diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index f19b8c7b1..3f1808c0f 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -1,7 +1,8 @@ //! `column_signs`, `depends_on_evaluation_time` and `scanned_sources`, //! derived once per memo group from its input groups (concepts doc, section //! 3, "Value sign" and `GroupProperties`; `aggregate_coverage` is in -//! `memo_aggregate_coverage.rs`). Every example reads one of three sources: +//! `memo_aggregate_coverage.rs`; column lineage, derived with the signs, +//! is in `column_lineage.rs`). Every example reads one of three sources: //! //! ```text //! latency [ts, value, host] value declared non-negative diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 4654835d5..84d7dd22e 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -48,7 +48,7 @@ This section fixes the words used below. They follow relational databases and Ap |---|---|---| | **Relation / table** | A set (bag) of rows with the same columns. A base table is stored; a derived relation is the output of a query operator. | Every edge in the DAG carries a relation. A `Scan` reads a base table (SQL table or PromQL metric); every other operator outputs a derived relation. | | **Row / tuple** | One element of a relation: one value per column. | One output row of a node. For PromQL, one sample of one series at one time. | -| **Column** | One position in every row, with a name and a type. Qualified as `table.column` when names can collide (DataFusion `Column { relation, name }`). | `ColumnId` refers to a column of the input schema; its memo group's column lineage, the source column `(source, column)` its values are copied from, identifies it across nodes (§4.2.2). A front end binds `table.column` with its own scope while it lowers. | +| **Column** | One position in every row, with a name and a type. Qualified as `table.column` when names can collide (DataFusion `Column { relation, name }`). | `ColumnId` refers to a column of the input schema; its memo group's column lineage, the source column `(source, scan ordinal, column)` its values are copied from or the memo group and column that computed them, identifies it across nodes (§4.2.2). A front end binds `table.column` with its own scope while it lowers. | | **Schema** | The ordered list of columns of a relation: name, data type, nullability (Arrow `Schema` of `Field { name, data_type, nullable }`; DataFusion `DFSchema` adds the table qualifier). The schema is *metadata*: it describes rows, it contains none. | `Schema` of `Field { name, dtype, nullable }` in `crates/types/src/ir/schema/mod.rs`, with no table qualifier. Unlike Arrow, `dtype` can be a summary state type (§3). | | **Data type** | The type of a column's values (`Int64`, `Utf8`, `Timestamp`, …). | `DataType`, wrapped as `FieldDataType::Plain`. | | **Aggregate state** | The intermediate value of an aggregate function before its final result, e.g. `(sum, count)` for `AVG` (DataFusion `Accumulator::state`, partial/final aggregation). It is never exposed as a column type to users. | Summary state *is* a column type here (`FieldDataType::Sketch`, `ExactAggregate`, …), so state can flow along edges and be merged, stored and read by later operators. | @@ -428,9 +428,10 @@ Any other shape stays in `definition`: **Which column a condition is on.** -- A column is named by its **lineage** ([concepts doc, section 3](../concepts/memo-groups-properties-requirements.md#column-lineage)) in the group the `SummaryAgg` reads: the source column `(source, column)` its values are copied from, a `Scan`'s table or metric and its column there. Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions, though both columns are named `region`. +- A column is named by its **lineage** (concepts doc, section 3, Column lineage) in the group the `SummaryAgg` reads: the source column `(source, scan ordinal, column)` its values are copied from, a `Scan`'s table or metric, which scan of it, and its column there. Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions, though both columns are named `region`. +- The scan ordinal tells a self-join's two scans of one source apart: 0 for the first scan, counted left to right across joins. In `FROM flows a JOIN flows b`, `a.proto` is `(flows, 0, proto)` and `b.proto` is `(flows, 1, proto)`; table aliases do not appear, so renaming them keeps the condition. - A rename keeps lineage: after `Project [region AS r]`, `r` is still `t.region`, so the condition is `t.region ∈ {us}`. A memo group's schema is derived, so no other operator renames a column. Since the `definition` contains the same `Project`, two states that rename the same way still compare equal. -- A condition on a computed column (a `Project` expression item, an aggregate's measure, a readout's value) has no source column to name and stays in `definition`. +- A condition on a computed column (a `Project` expression item, an aggregate's measure, a readout's value) has no source column to name and stays in `definition`. Its lineage is `Computed { group, column }`, the memo group and output column that computed it, so two computed columns have equal lineage only when they are the same computation. - If two columns the `SummaryAgg` reads have the same lineage (for example `Project [a AS k, a AS k2]`), the selection cannot name one of them apart, and their conditions stay in `definition`. - A PromQL label is named by its metric and label: `latency.job`. - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. @@ -1354,7 +1355,7 @@ pub struct CurrentSeriesInput { A finalized value's accuracy statement is `AccuracyGuarantee` (`accuracy/guarantee.rs`): a `spec` (`AccuracySpec`) with `accuracy_metric` (which error is measured, e.g. rank error), `error` (the error bound), `failure_probability` (the chance the bound does not hold) and `accuracy_scope` (which output values one statement covers), plus `provenance` (which estimates the bound came from). It is attached to evaluation and finalized nodes, and to `MaintainPopulation` nodes (whose population is exact); never to sketch or accumulator state. -### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/column_properties.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) +### 6.6 Aggregate coverage (`crates/types/src/stages/logical/aggregate_coverage.rs`; selection algebra in `crates/types/src/ir/properties/selection.rs`; `SourceColumn` in `crates/types/src/ir/operator/arguments.rs`; column lineage (`ColumnLineage`, a memo-group property) in `crates/types/src/stages/logical/column_properties.rs`; the plan's content id in `crates/types/src/stages/logical/view_coverage.rs`) The code for §4. `AggregateDefinition` is shown in §4.2.2. @@ -1383,10 +1384,21 @@ pub struct SelectionBox { pub struct SourceColumn { /// The table or metric a `Scan` reads. pub source: Source, + /// Which scan of `source`: 0 for the first, counted left to right + /// across joins (a self-join's right side is 1). + pub scan_ordinal: u32, /// The column's name in that source. pub column: String, } +/// Where a memo group's output column's values come from. +pub enum ColumnLineage { + /// Copied unchanged from a column of one scan of a source. + SourceColumn(SourceColumn), + /// Column `column` of memo group `group`, where it was computed. + Computed { group: MemoGroupId, column: ColumnId }, +} + /// A constraint on one column. pub enum Constraint { /// The value is one of these. From 158d4d5f3d40f923849ff0130655cad37f964196 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:46:11 +0000 Subject: [PATCH 3/5] fix(types): BinaryOp labels come from the output side; only Join shifts scan ordinals A BinaryOp does not concatenate both sides' columns: its labels are the left side's for one-to-one and group_left, the right ("many") side's for group_right, and a matched label has equal values on both sides, so it takes the output side's lineage and sign (`or` keeps what both sides agree on). No scan ordinal shifts at a BinaryOp; only a Join's right side shifts. Labels named in group_left/group_right still come from the one side. Tests: rate(x[5m]) / rate(x[1h]) keeps job as x scan 0; a same-source group_right takes the right side's labels (failed before this change). --- .../src/stages/logical/column_properties.rs | 57 ++++++++++-------- crates/types/tests/column_lineage.rs | 59 ++++++++++++++++++- 2 files changed, 89 insertions(+), 27 deletions(-) diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index 4d381154a..07616d315 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -155,8 +155,8 @@ fn columns_of(input: &Input<'_>) -> Vec { .collect() } -/// `right`, the right side's columns of an operator that concatenates two -/// inputs' columns after `left`'s: a source column of a source `left` also +/// `right`, the right side's columns of a `Join`, which concatenates them +/// after `left`'s: a source column of a source `left` also /// scans moves to the scans after `left`'s (its scan ordinal shifted by /// `left`'s largest for that source, plus one), so a self-join's two sides /// name different scans. @@ -282,23 +282,20 @@ fn non_asap_columns( columns.push(ColumnProperties::computed(Unknown)); columns } - // The left side's columns, its value computed per the operator, - // then the right side's labels the match adds. A label that may - // come from either side (`or`, `group_right`) keeps what both sides - // agree on; a label `group_left(...)` names comes from the right - // ("one") side and one `group_right(...)` names from the left, - // overwriting a column of that name. A vector match is a join, so - // the right side reads scans after the left's; `or` is a union. + // The output side's columns (the left's; the right's, the "many" + // side, for `group_right`), its value computed per the operator, + // then the right side's labels the match adds. A matched label has + // equal values on both sides, so it takes the output side's lineage + // and sign; `or`, a union, keeps what both sides agree on. A label + // `group_left(...)` names comes from the right ("one") side and one + // `group_right(...)` names from the left, overwriting a column of + // that name. The output does not concatenate both sides' columns, + // so no scan ordinal shifts. BinaryOp { operator, lhs, rhs, .. } => { - let is_or = matches!(operator.kind, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)); let left = columns_of(lhs); - let right = if is_or { - columns_of(rhs) - } else { - after_left_scans(&left, columns_of(rhs)) - }; + let right = columns_of(rhs); let named = |input: &Input<'_>, columns: &[ColumnProperties], name: &str| { input .schema() @@ -310,17 +307,25 @@ fn non_asap_columns( .vector_match .as_ref() .and_then(|m| m.grouping.as_ref()); - let right_rows = - is_or || grouping.is_some_and(|grouping| grouping.side == GroupSide::Right); - let mut columns = left.clone(); - if right_rows { - for (column, field) in lhs.schema().fields.iter().enumerate() { - columns[column] = - columns[column] - .clone() - .agreed_with(&named(rhs, &right, &field.name)); - } - } + let output_side_is_right = + grouping.is_some_and(|grouping| grouping.side == GroupSide::Right); + let is_or = matches!(operator.kind, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)); + let mut columns: Vec = lhs + .schema() + .fields + .iter() + .enumerate() + .map(|(column, field)| { + let from_right = || named(rhs, &right, &field.name); + if output_side_is_right { + from_right() + } else if is_or { + left[column].clone().agreed_with(&from_right()) + } else { + left[column].clone() + } + }) + .collect(); if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { let left_sign = left[value].sign; let right_sign = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs index dc324825a..2f69536ea 100644 --- a/crates/types/tests/column_lineage.rs +++ b/crates/types/tests/column_lineage.rs @@ -23,7 +23,7 @@ use asap_types::stages::{ LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, ValueSign, }; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; -use asap_types::test_support::memo_fixtures::{filter, range, shift}; +use asap_types::test_support::memo_fixtures::{aggregate, filter, range, shift}; use asap_types::workload::{DataWorkload, SourceFacts}; fn memo() -> Memo { @@ -678,3 +678,60 @@ fn a_group_right_label_comes_from_the_left_side() { (series_column("a", "team"), ValueSign::NonNegative) ); } + +/// `rate(x[])`: per series, `[ts, value, job, team]` kept in place. +fn rate_of_x(memo: &mut Memo, width_ms: i64) -> MemoGroupId { + let x = memo + .find_or_add_group(series("x", &["job", "team"])) + .unwrap(); + let window = memo.find_or_add_group(range(width_ms, x)).unwrap(); + memo.find_or_add_group(aggregate(Reduction::PerEntity, AggIntent::Rate, window)) + .unwrap() +} + +/// `rate(x[5m]) / rate(x[1h])`: both sides read `x`, but a `BinaryOp` +/// does not concatenate their columns; its labels are the left side's, so +/// `job` is still `x` scan 0's. +#[test] +fn a_same_source_binary_op_keeps_its_labels_scan() { + let mut memo = match_grouping_memo(); + let short = rate_of_x(&mut memo, 300_000); + let long = rate_of_x(&mut memo, 3_600_000); + let ratio = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + }, + return_bool: false, + lhs: short, + rhs: long, + })) + .unwrap(); + assert_eq!( + lineage_and_sign_of(&memo, ratio, "job").0, + series_column("x", "job") + ); +} + +/// `rate(x[5m]) * on(job) group_right(team) rate(x[1h])`: the output's rows +/// are the right ("many") side's, so its labels take that side's lineage, +/// `x` scan 0's, not a shifted scan or a disagreement; `team` comes from +/// the left ("one") side, also `x` scan 0's. +#[test] +fn a_same_source_group_right_takes_the_right_sides_labels() { + let mut memo = match_grouping_memo(); + let short = rate_of_x(&mut memo, 300_000); + let long = rate_of_x(&mut memo, 3_600_000); + let product = memo + .find_or_add_group(grouped_product(GroupSide::Right, short, long)) + .unwrap(); + for label in ["job", "team"] { + assert_eq!( + lineage_and_sign_of(&memo, product, label).0, + series_column("x", label) + ); + } +} From a6c1ffe51a99eef4ac71b9d55439ee0ed6d2e3c4 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:09:54 +0000 Subject: [PATCH 4/5] refactor(types, frontends): lineage review round 2: group id only where computed, Option lineage in coverage, resolver qualifiers per column - The memo group id reaches only column_lineage_and_signs (via derive_group_properties); project_lineage returns Option lineage (None: computed here), which lineage_at_top and source_column read, so aggregate_coverage, validate_expression_over_input_groups and the coverage helpers lose the group parameter. - ColumnProperties: Default (computed, sign Unknown) and all_computed replace the repeated computed(Unknown) forms; Debug/PartialEq dropped; ColumnLineage drops Eq. - BinaryOp `or`: a label only the right side has is computed (null on the left side's rows). - regrouped key check reads with matches!. - Deleted: resolve_column_refs (unused) and the planner's SourceColumn re-export (unused). - Front-end resolver: Bound stores one qualifier per column (None when unqualified); the padding closure, the window-function check and the empty-list sentinel go; one Scope::qualifier accessor; pass-through operators move the child node. An aggregate's qualifiers are sized by its node's schema. - Tests: memo_fixtures::series_column / table_column replace the local lineage helpers; latency_series and series use memo_fixtures::scan; new test an_or_label_only_the_right_side_has_is_computed. - Docs: drop the stale Field.table and Project qualifier from pre-asap-ir and operator-sharing; SourceColumn doc no longer names crate::stages. --- crates/frontend-common/src/resolve.rs | 139 ++++++------- crates/planner/src/lib.rs | 1 - crates/types/src/ir/operator/arguments.rs | 5 +- .../types/src/ir/scalar/column_resolution.rs | 14 +- crates/types/src/ir/scalar/mod.rs | 2 +- .../src/stages/logical/aggregate_coverage.rs | 66 +++--- .../src/stages/logical/column_properties.rs | 121 +++++------ crates/types/src/stages/logical/group.rs | 8 +- crates/types/src/stages/logical/memo/mod.rs | 2 +- .../types/src/test_support/memo_fixtures.rs | 24 ++- crates/types/tests/column_lineage.rs | 195 +++++++++--------- crates/types/tests/memo_aggregate_coverage.rs | 41 ++-- .../design_docs/proposals/operator-sharing.md | 2 +- docs/develop_docs/pre-asap-ir.md | 9 +- 14 files changed, 306 insertions(+), 323 deletions(-) diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 77ac7ab14..1904b798c 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -45,8 +45,9 @@ use crate::canonicalize::canonicalize; /// The names an operator's parent resolves against: the operator's output /// schema and, per column, the SQL table or alias that qualifies it (`None` -/// when unqualified; an empty list qualifies no column). A qualified -/// reference (`f.src`) binds here, while lowering; the IR keeps no qualifier. +/// when unqualified; [`Scope::unqualified`]'s empty list qualifies no +/// column). A qualified reference (`f.src`) binds here, while lowering; the +/// IR keeps no qualifier. #[derive(Clone, Copy)] struct Scope<'a> { schema: &'a Schema, @@ -76,7 +77,7 @@ impl<'a> Scope<'a> { .fields .iter() .enumerate() - .filter(|(id, field)| field.name == *name && self.qualifier_ref(*id) == wanted) + .filter(|(id, field)| field.name == *name && self.qualifier(*id) == wanted) .map(|(id, _)| id) .collect::>() }; @@ -103,11 +104,7 @@ impl<'a> Scope<'a> { } /// The qualifier of column `column`. - fn qualifier(&self, column: ColumnId) -> Option { - self.qualifier_ref(column).map(str::to_owned) - } - - fn qualifier_ref(&self, column: ColumnId) -> Option<&str> { + fn qualifier(&self, column: ColumnId) -> Option<&'a str> { self.qualifiers.get(column).and_then(Option::as_deref) } } @@ -120,19 +117,19 @@ struct Bound { } impl Bound { - /// `node` with `qualifiers`: one per column, or none (unqualified). + /// `node` with `qualifiers`, one per column. fn new(node: Rc, qualifiers: Vec>) -> Self { - debug_assert!( - qualifiers.is_empty() || qualifiers.len() == node.schema.fields.len(), - "{} qualifiers for {} columns", + debug_assert_eq!( qualifiers.len(), - node.schema.fields.len() + node.schema.fields.len(), + "one qualifier per column" ); Self { node, qualifiers } } fn unqualified(node: Rc) -> Self { - Self::new(node, Vec::new()) + let qualifiers = vec![None; node.schema.fields.len()]; + Self::new(node, qualifiers) } fn scope(&self) -> Scope<'_> { @@ -204,8 +201,9 @@ fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result, _>>() }; - // A node over `child` with `child`'s columns and qualifiers. - let passing = |op: NonASAPOp, child: Bound| Ok(Bound::new(node(op)?, child.qualifiers)); + // A node over a child with the child's columns and `qualifiers`. + let passing = + |op: NonASAPOp, qualifiers: Vec>| Ok(Bound::new(node(op)?, qualifiers)); let unqualified = |op: NonASAPOp| Ok(Bound::unqualified(node(op)?)); // A PromQL child over an open schema gets an opaque series-identity column. let promql_series = |child: &UnresolvedOp| -> Result, ResolveDAGError> { @@ -314,9 +312,9 @@ fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result Result Result Result, _>>()?; // A `by` key keeps its qualifier; a per-series reduction keeps - // its input's columns in place. A top-k emits ranked rows. + // its input's columns in place. A top-k emits ranked rows, and a + // measure that adds label columns (`count_values`, `absent`) + // breaks the keys-then-measures layout: no column is qualified. let out_schema = aggregate_output_schema(&child.node.schema, &reduction, &measures, output_names)?; - let qualifiers: Vec> = match &reduction { - _ if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) => Vec::new(), - Reduction::PerEntity => child.qualifiers.clone(), - Reduction::Reduce(by) if by.is_without() => Vec::new(), - Reduction::Reduce(by) => by - .keys() - .iter() - .map(|&key| scope.qualifier(key)) - .chain(std::iter::repeat_n(None, measures.len())) - .collect(), + let keyed: Option>> = match &reduction { + _ if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) => None, + Reduction::PerEntity => Some(child.qualifiers.clone()), + Reduction::Reduce(by) if by.is_without() => None, + Reduction::Reduce(by) => Some( + by.keys() + .iter() + .map(|&key| scope.qualifier(key).map(str::to_owned)) + .chain(std::iter::repeat_n(None, measures.len())) + .collect(), + ), }; - // A measure that adds label columns (`count_values`, `absent`) - // breaks the keys-then-measures layout: then no column is - // qualified. - let qualifiers = if qualifiers.len() == out_schema.fields.len() { - qualifiers - } else { - Vec::new() + let qualifiers_of = |columns: usize| { + keyed + .clone() + .filter(|keyed| keyed.len() == columns) + .unwrap_or_else(|| vec![None; columns]) }; // HAVING is evaluated over the aggregate's own output. let having = having @@ -420,22 +419,21 @@ fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result { @@ -444,9 +442,9 @@ fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result Result>() - }; let mut concat = left.node.schema.clone(); concat .fields .extend(right.node.schema.fields.iter().cloned()); - let mut qualifiers = qualify(&left); - qualifiers.extend(qualify(&right)); + let mut qualifiers = left.qualifiers; + qualifiers.extend(right.qualifiers); let pred = pred( &p.0, Scope { @@ -540,9 +533,9 @@ fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result Result Result Result Result Result Result { match col { // A name must be unique: after a join `k` could be either side's. - // A schema carries no qualifier; the front end binds a qualified - // name with its own scope before it calls this. + // A schema carries no qualifier, so a qualified name that reaches + // here (an extension intent's payload column, a SQL temporal + // aggregate's argument) resolves by its unique name; the front end + // binds every other qualified name with its own scope. ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => { match schema.fields.iter().filter(|c| &c.name == name).count() { 0 => Err(ResolveError::NotFound { @@ -72,14 +74,6 @@ pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result Result, ResolveError> { - cols.iter().map(|c| resolve_column_ref(c, schema)).collect() -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index ce7644b65..234717316 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -12,7 +12,7 @@ pub mod column_resolution; pub mod expr_ir; pub mod scalar_type_rules; -pub use column_resolution::{resolve_column_ref, resolve_column_refs, ResolveError}; +pub use column_resolution::{resolve_column_ref, ResolveError}; pub use expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use std::rc::Rc; diff --git a/crates/types/src/stages/logical/aggregate_coverage.rs b/crates/types/src/stages/logical/aggregate_coverage.rs index 8719428e7..a14e9552b 100644 --- a/crates/types/src/stages/logical/aggregate_coverage.rs +++ b/crates/types/src/stages/logical/aggregate_coverage.rs @@ -160,9 +160,7 @@ fn stored_coverage<'m>(input: &MemoGroupAsInput<'m>) -> Option<&'m AggregateCove /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may /// also merge overlapping selections, and a counter family /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. -/// `group` is the memo group `operator` computes. pub(super) fn aggregate_coverage<'m>( - group: MemoGroupId, operator: &Operator>, ) -> Result { let as_built = |op: &ASAPOp>| -> ASAPOp> { @@ -172,7 +170,7 @@ pub(super) fn aggregate_coverage<'m>( Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { let top = unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); - of_aggregate(group, &InputOrTemporaryBuild::Input(*child), None, |_| top) + of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) } Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) if operator.output_kind() == OperatorResultKind::State => @@ -180,12 +178,12 @@ pub(super) fn aggregate_coverage<'m>( of_window_selection(operator) } Operator::NonASAP(_) => match selected_build(operator)? { - Some(build) => of_summary_agg(group, &build), + Some(build) => of_summary_agg(&build), None => Err(CoverageError::NoCoverage), }, Operator::ASAP( op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), - ) => of_summary_agg(group, &as_built(op)), + ) => of_summary_agg(&as_built(op)), Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { @@ -371,8 +369,8 @@ fn regrouped( .fields .get(key) .is_some_and(|field| field.is_plain()) - && state.column_lineage.get(key).is_some() - && state.column_lineage.get(key) == rows_lineage.get(column) + && matches!(state.column_lineage.get(key), + Some(lineage) if Some(lineage) == rows_lineage.get(column)) }) .ok_or(CoverageError::NotAGroupKey { key }) }) @@ -419,20 +417,20 @@ fn build_input<'m>(state: &MemoGroupAsInput<'m>) -> Option> /// reads ([`build_input`]), as a selection names it. fn time_column_of_build_input(state: &MemoGroupAsInput<'_>) -> Option { let rows = build_input(state)?; - source_column( - &rows.group_properties().column_lineage, - rows.schema().time_index?, - ) + let lineage: Vec> = rows + .group_properties() + .column_lineage + .iter() + .cloned() + .map(Some) + .collect(); + source_column(&lineage, rows.schema().time_index?) } /// The coverage of `build`, a `SummaryAgg` or `SummaryTumblingWindowChain` /// over inputs (or over an input a rule built over them): a chain covers -/// what its `SummaryAgg` over the same input would. `group` is the memo -/// group being derived. -fn of_summary_agg( - group: MemoGroupId, - build: &ASAPOp>, -) -> Result { +/// what its `SummaryAgg` over the same input would. +fn of_summary_agg(build: &ASAPOp>) -> Result { let (ASAPOp::SummaryAgg { child, summary_type: family, @@ -451,7 +449,7 @@ fn of_summary_agg( else { unreachable!("called on a SummaryAgg or a chain"); }; - of_aggregate(group, child, filter.as_ref(), |residual_filter| { + of_aggregate(child, filter.as_ref(), |residual_filter| { Operator::ASAP(ASAPOp::SummaryAgg { child: None, summary_type: family.clone(), @@ -466,9 +464,8 @@ fn of_summary_agg( /// down through the liftable operators of its inputs' operators. /// `filter` is the aggregate's own row filter (a `SummaryAgg`'s), and /// `unlifted_top` makes the aggregate's unlifted operator from that filter's -/// residual. `group` is the memo group being derived. +/// residual. fn of_aggregate<'m>( - group: MemoGroupId, child: &Built<'m>, filter: Option<&Predicate>>, unlifted_top: impl FnOnce(Option>>) -> Operator>, @@ -500,7 +497,7 @@ fn of_aggregate<'m>( let InputOrTemporaryBuild::Input(base_input) = &base else { unreachable!("a rule builds only liftable operators over real inputs"); }; - let top = lineage_at_top(group, &base_input.group_properties().column_lineage, &chain); + let top = lineage_at_top(&base_input.group_properties().column_lineage, &chain); let top = top.as_slice(); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. @@ -600,7 +597,7 @@ fn of_aggregate<'m>( /// bottom. A conjunct on a column [`source_column`] cannot name stays. fn residual( pred: &Predicate>, - top: &[ColumnLineage], + top: &[Option], above: &[Operator>], lifted: &mut BTreeMap, ) -> Option>> { @@ -667,29 +664,24 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option( - group: MemoGroupId, - base: &[ColumnLineage], - chain: &[Operator], -) -> Vec { - let mut lineage = base.to_vec(); +/// `Project`'s is [`project_lineage`]'s. `None` is a column computed in the +/// chain, which no selection names. +fn lineage_at_top(base: &[ColumnLineage], chain: &[Operator]) -> Vec> { + let mut lineage: Vec> = base.iter().cloned().map(Some).collect(); for link in chain.iter().rev() { if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { - lineage = project_lineage(group, cols, &lineage); + lineage = project_lineage(cols, &lineage); } } lineage } /// The source column a selection names column `column` by: its lineage, one -/// of `lineage` (per column). `None` when the column is `Computed`, or -/// another column has the same lineage (`Project [a AS k, a AS k2]`), so a -/// selection cannot name it apart. -fn source_column(lineage: &[ColumnLineage], column: ColumnId) -> Option { - let ColumnLineage::SourceColumn(source_column) = lineage.get(column)? else { +/// of `lineage` (per column; `None` computed). `None` when the column is +/// computed, or another column has the same lineage (`Project [a AS k, a AS +/// k2]`), so a selection cannot name it apart. +fn source_column(lineage: &[Option], column: ColumnId) -> Option { + let Some(ColumnLineage::SourceColumn(source_column)) = lineage.get(column)? else { return None; }; let namesakes = lineage diff --git a/crates/types/src/stages/logical/column_properties.rs b/crates/types/src/stages/logical/column_properties.rs index 07616d315..fe500ae94 100644 --- a/crates/types/src/stages/logical/column_properties.rs +++ b/crates/types/src/stages/logical/column_properties.rs @@ -60,7 +60,7 @@ pub enum ValueSign { /// they hold the same values: the same column of the same scan, or the /// same computation. Aggregate coverage names a selection's column by its /// source column. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq)] pub enum ColumnLineage { /// Copied unchanged from a column of one scan of a source. SourceColumn(SourceColumn), @@ -79,7 +79,7 @@ use ValueSign::{NonNegative, NonPositive, Unknown}; type Input<'m> = MemoGroupAsInput<'m>; /// One output column's lineage and sign. -#[derive(Debug, Clone, PartialEq)] +#[derive(Clone)] struct ColumnProperties { /// `None` for a column the group being derived computes: its lineage /// is `Computed` by that group at the column's final position. @@ -87,6 +87,18 @@ struct ColumnProperties { sign: ValueSign, } +/// A column the operator computes, with no sign known. +impl Default for ColumnProperties { + fn default() -> Self { + Self::computed(Unknown) + } +} + +/// `columns` columns the operator computes, with no sign known. +fn all_computed(columns: usize) -> Vec { + vec![ColumnProperties::default(); columns] +} + impl ColumnProperties { /// A column the operator computes, with values of `sign`. fn computed(sign: ValueSign) -> Self { @@ -119,7 +131,7 @@ pub(super) fn column_lineage_and_signs( source_facts: &BTreeMap, ) -> (Vec, Vec) { let columns = match operator { - Operator::NonASAP(op) => non_asap_columns(group, op, output, source_facts), + Operator::NonASAP(op) => non_asap_columns(op, output, source_facts), Operator::ASAP(op) => asap_columns(op, output), }; debug_assert_eq!( @@ -182,13 +194,11 @@ fn after_left_scans( } fn non_asap_columns( - group: MemoGroupId, op: &NonASAPOp>, output: &Schema, source_facts: &BTreeMap, ) -> Vec { use NonASAPOp::*; - let unknown = || vec![ColumnProperties::computed(Unknown); output.fields.len()]; match op { // Each column copied from its source, with the sign its source's // facts declare (a histogram's or summary's series reads its @@ -219,10 +229,10 @@ fn non_asap_columns( }) .collect() } - Values { .. } => unknown(), + Values { .. } => all_computed(output.fields.len()), // The scalar's value, every other column `Unknown`. PromqlVectorFromScalar(expr) => { - let mut columns = unknown(); + let mut columns = all_computed(output.fields.len()); if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, output) { columns[value] = ColumnProperties::computed(expression_sign(expr, &[])); } @@ -240,14 +250,14 @@ fn non_asap_columns( // The relabeled column is written; every other column is copied. PromqlRelabel { dst, child, .. } => { let mut columns = columns_of(child); - let written = ColumnProperties::computed(Unknown); + let written = ColumnProperties::default(); match child.schema().column_id(dst) { Some(column) => columns[column] = written, None => columns.push(written), } columns } - Project { cols, child, .. } => project_columns(group, cols, child), + Project { cols, child, .. } => project_columns(cols, child), Aggregate { reduction, measures, @@ -259,8 +269,8 @@ fn non_asap_columns( // The ranked rows, projected: each column copied. let name = topk_output_name(*k, output_names); return match ranked_row_projection(child, reduction, &name) { - Ok(cols) => project_columns(group, &cols, child), - Err(_) => unknown(), + Ok(cols) => project_columns(&cols, child), + Err(_) => all_computed(output.fields.len()), }; } aggregate_columns(child, reduction, measures, output) @@ -279,14 +289,15 @@ fn non_asap_columns( Concat { children } => agreed(&children.iter().map(columns_of).collect::>()), SQLWindowFunc { child, .. } => { let mut columns = columns_of(child); - columns.push(ColumnProperties::computed(Unknown)); + columns.push(ColumnProperties::default()); columns } // The output side's columns (the left's; the right's, the "many" // side, for `group_right`), its value computed per the operator, // then the right side's labels the match adds. A matched label has // equal values on both sides, so it takes the output side's lineage - // and sign; `or`, a union, keeps what both sides agree on. A label + // and sign; `or`, a union, keeps what both sides agree on (a label + // one side lacks is computed). A label // `group_left(...)` names comes from the right ("one") side and one // `group_right(...)` names from the left, overwriting a column of // that name. The output does not concatenate both sides' columns, @@ -301,7 +312,7 @@ fn non_asap_columns( .schema() .column_id(name) .and_then(|column| columns.get(column).cloned()) - .unwrap_or(ColumnProperties::computed(Unknown)) + .unwrap_or_default() }; let grouping = operator .vector_match @@ -343,8 +354,16 @@ fn non_asap_columns( BinaryOpKind::Arithmetic(op) => arithmetic_sign(op, left_sign, right_sign), }); } + // Under `or` a label only the right side has is null on the left + // side's rows: computed, like a label only the left side has. let added = output.fields.get(columns.len()..).unwrap_or_default(); - columns.extend(added.iter().map(|field| named(rhs, &right, &field.name))); + columns.extend(added.iter().map(|field| { + if is_or { + ColumnProperties::default() + } else { + named(rhs, &right, &field.name) + } + })); columns.truncate(output.fields.len()); if let Some(grouping) = grouping { let (one_side, one_side_columns) = match grouping.side { @@ -364,7 +383,6 @@ fn non_asap_columns( fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec { use ASAPOp::*; - let unknown = || vec![ColumnProperties::computed(Unknown); output.fields.len()]; match op { SummaryAgg { child, @@ -373,7 +391,7 @@ fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec state_columns(child, summary_type, input, reduction) - .map_or_else(unknown, |(_, columns)| columns), + .map_or_else(|| all_computed(output.fields.len()), |(_, columns)| columns), // The build's layout without its own time column, after the // computed `window_end` (column 0). SummaryTumblingWindowChain { @@ -387,10 +405,10 @@ fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec unknown(), + None => all_computed(output.fields.len()), }, SummaryMerge { children, group_by } => { // The windows a `TimeRange` over a chain selects merge into the @@ -405,7 +423,7 @@ fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec>()); @@ -414,12 +432,7 @@ fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec { - let column_at = |column: usize| { - merged - .get(column) - .cloned() - .unwrap_or(ColumnProperties::computed(Unknown)) - }; + let column_at = |column: usize| merged.get(column).cloned().unwrap_or_default(); let state = children[0] .schema() .fields @@ -469,45 +482,38 @@ fn asap_columns(op: &ASAPOp>, output: &Schema) -> Vec unknown(), + | Extension { .. } => all_computed(output.fields.len()), } } -/// The columns of `cols`, a `Project` computed by memo group `group`, over -/// `child`: lineage by [`project_lineage`], sign by [`expression_sign`]. -fn project_columns( - group: MemoGroupId, - cols: &[ProjectItem], - child: &Input<'_>, -) -> Vec { +/// The columns of `cols`, a `Project` over `child`: lineage by +/// [`project_lineage`], sign by [`expression_sign`]. +fn project_columns(cols: &[ProjectItem], child: &Input<'_>) -> Vec { let input = child.group_properties(); - project_lineage(group, cols, &input.column_lineage) + let input_lineage: Vec> = + input.column_lineage.iter().cloned().map(Some).collect(); + project_lineage(cols, &input_lineage) .into_iter() .zip(cols) .map(|(lineage, item)| ColumnProperties { - lineage: Some(lineage), + lineage, sign: expression_sign(&item.expr, &input.column_signs), }) .collect() } -/// The lineage of the output columns of `cols`, a `Project` computed by -/// memo group `group` over input columns of lineage `input`: a column -/// item, renamed or not, copies its input column's; an expression item is -/// `Computed` by `group` at its position. +/// The lineage of the output columns of `cols`, a `Project` over input +/// columns of lineage `input` (`None`: computed by the group being +/// derived): a column item, renamed or not, copies its input column's; an +/// expression item is `None`, computed by the `Project`'s group. pub(super) fn project_lineage( - group: MemoGroupId, cols: &[ProjectItem], - input: &[ColumnLineage], -) -> Vec { + input: &[Option], +) -> Vec> { cols.iter() - .enumerate() - .map(|(column, item)| { - match item.expr { - ScalarExpr::Column(input_column) => input.get(input_column).cloned(), - _ => None, - } - .unwrap_or(ColumnLineage::Computed { group, column }) + .map(|item| match item.expr { + ScalarExpr::Column(column) => input.get(column).cloned().flatten(), + _ => None, }) .collect() } @@ -550,7 +556,7 @@ fn aggregate_columns( output: &Schema, ) -> Vec { let Ok((_, sources)) = aggregate_output(child.schema(), reduction, measures, &[]) else { - return vec![ColumnProperties::computed(Unknown); output.fields.len()]; + return all_computed(output.fields.len()); }; columns_from_sources(child, &sources, |index, input| { measure_sign(&measures[index], input) @@ -569,7 +575,7 @@ fn columns_from_sources( let column_at = |column: Option| { column .and_then(|column| input.get(column).cloned()) - .unwrap_or(ColumnProperties::computed(Unknown)) + .unwrap_or_default() }; sources .iter() @@ -579,7 +585,7 @@ fn columns_from_sources( index, input_column, } => ColumnProperties::computed(measure(*index, column_at(*input_column).sign)), - OutputColumnSource::AddedLabel => ColumnProperties::computed(Unknown), + OutputColumnSource::AddedLabel => ColumnProperties::default(), }) .collect() } @@ -645,7 +651,7 @@ fn readout_columns( ) -> Vec { let input = state.schema(); if input.fields.len() != output.fields.len() { - return vec![ColumnProperties::computed(Unknown); output.fields.len()]; + return all_computed(output.fields.len()); } columns_of(state) .into_iter() @@ -697,12 +703,7 @@ fn agreed(inputs: &[Vec]) -> Vec { .enumerate() .map(|(column, properties)| { rest.iter().fold(properties.clone(), |agreed, other| { - agreed.agreed_with( - &other - .get(column) - .cloned() - .unwrap_or(ColumnProperties::computed(Unknown)), - ) + agreed.agreed_with(&other.get(column).cloned().unwrap_or_default()) }) }) .collect() diff --git a/crates/types/src/stages/logical/group.rs b/crates/types/src/stages/logical/group.rs index 78992caa8..4fdde2aaf 100644 --- a/crates/types/src/stages/logical/group.rs +++ b/crates/types/src/stages/logical/group.rs @@ -108,18 +108,16 @@ pub enum MemoGroupError { /// Whether `operator` is a valid expression over its input groups /// ([`MemoGroupAsInput`]), never a node tree: its output schema derives, its /// inputs pass validation, and a merge reads only states it may merge. -/// `group` is the memo group the expression computes (new or existing). /// Returns the output schema and, for a merge, the coverage its check /// derived. pub(super) fn validate_expression_over_input_groups( - group: MemoGroupId, operator: &Operator>, ) -> Result<(Schema, Option), MemoGroupError> { let schema = operator.output_schema()?; operator.validate_inputs()?; // A merge is valid only over states of one definition it may merge. let merge_coverage = match operator { - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(aggregate_coverage(group, operator)?), + Operator::ASAP(ASAPOp::SummaryMerge { .. }) => Some(aggregate_coverage(operator)?), _ => None, }; Ok((schema, merge_coverage)) @@ -135,8 +133,8 @@ pub(super) fn derive_group_properties( expression: &Operator, ) -> Result { let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); - let (schema, merge_coverage) = validate_expression_over_input_groups(group, &operator)?; - let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(group, &operator).ok()); + let (schema, merge_coverage) = validate_expression_over_input_groups(&operator)?; + let aggregate_coverage = merge_coverage.or_else(|| aggregate_coverage(&operator).ok()); let (column_lineage, column_signs) = column_lineage_and_signs(group, &operator, &schema, &memo.source_facts); Ok(GroupProperties { diff --git a/crates/types/src/stages/logical/memo/mod.rs b/crates/types/src/stages/logical/memo/mod.rs index 957897109..69443a586 100644 --- a/crates/types/src/stages/logical/memo/mod.rs +++ b/crates/types/src/stages/logical/memo/mod.rs @@ -182,7 +182,7 @@ impl Memo { memo: self, group_id, }); - let (schema, _) = validate_expression_over_input_groups(group, &operator) + let (schema, _) = validate_expression_over_input_groups(&operator) .map_err(CandidateError::NotValidOverInputGroups)?; let result_kind = operator.output_kind(); let group_properties = &self.groups[group.0 as usize].group_properties; diff --git a/crates/types/src/test_support/memo_fixtures.rs b/crates/types/src/test_support/memo_fixtures.rs index 6fc9b4d40..5da450b33 100644 --- a/crates/types/src/test_support/memo_fixtures.rs +++ b/crates/types/src/test_support/memo_fixtures.rs @@ -3,7 +3,7 @@ //! `memo_aggregate_coverage`) build their examples from. use std::time::Duration; -use crate::ir::operator::arguments::{AtModifier, Reduction, Source, TimeShift}; +use crate::ir::operator::arguments::{AtModifier, Reduction, Source, SourceColumn, TimeShift}; use crate::ir::operator::AggIntent; use crate::ir::scalar::ColumnRef; use crate::ir::schema::{DataType, Field, FieldDataType, Schema}; @@ -37,6 +37,28 @@ pub fn scan( }) } +/// Column `column` of scan `scan_ordinal` of the series `metric`. +pub fn series_column(metric: &str, scan_ordinal: u32, column: &str) -> SourceColumn { + SourceColumn { + source: Source::TimeSeries { + metric: metric.into(), + }, + scan_ordinal, + column: column.into(), + } +} + +/// Column `column` of scan `scan_ordinal` of the table `table`. +pub fn table_column(table: &str, scan_ordinal: u32, column: &str) -> SourceColumn { + SourceColumn { + source: Source::Table { + table_ref: table.into(), + }, + scan_ordinal, + column: column.into(), + } +} + pub fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { Operator::NonASAP(NonASAPOp::Filter { pred: Predicate(pred), diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs index 2f69536ea..94bc495db 100644 --- a/crates/types/tests/column_lineage.rs +++ b/crates/types/tests/column_lineage.rs @@ -7,8 +7,8 @@ use std::collections::BTreeMap; use asap_types::ir::operator::arguments::{ - AtModifier, BinaryOpKind, GroupSide, JoinKind, Reduction, Source, SourceColumn, VectorGrouping, - VectorMatch, VectorMatchKind, + AtModifier, BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, Source, + SourceColumn, VectorGrouping, VectorMatch, VectorMatchKind, }; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::selection::Constraint; @@ -23,7 +23,9 @@ use asap_types::stages::{ LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, ValueSign, }; use asap_types::summary_type::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; -use asap_types::test_support::memo_fixtures::{aggregate, filter, range, shift}; +use asap_types::test_support::memo_fixtures::{ + self, aggregate, filter, range, series_column, shift, table_column, +}; use asap_types::workload::{DataWorkload, SourceFacts}; fn memo() -> Memo { @@ -67,37 +69,6 @@ fn hll(child: MemoGroupId, keys: Vec, column: &str) -> Operator SourceColumn { - column_of_scan(table, 0, column) -} - -/// The column `column` of scan `scan_ordinal` of the table `table`. -fn column_of_scan(table: &str, scan_ordinal: u32, column: &str) -> SourceColumn { - SourceColumn { - source: Source::Table { - table_ref: table.into(), - }, - scan_ordinal, - column: column.into(), - } -} - -fn source_column(table: &str, column: &str) -> ColumnLineage { - ColumnLineage::SourceColumn(column_of(table, column)) -} - -/// The column `column` of the series `latency`. -fn latency_column(column: &str) -> ColumnLineage { - ColumnLineage::SourceColumn(SourceColumn { - source: Source::TimeSeries { - metric: "latency".into(), - }, - scan_ordinal: 0, - column: column.into(), - }) -} - /// Column `column` of memo group `group`, where it was computed. fn computed(group: MemoGroupId, column: usize) -> ColumnLineage { ColumnLineage::Computed { group, column } @@ -140,7 +111,8 @@ fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { query: SketchStatistic::Cardinality, })) .unwrap(); - let flows = ["src", "dst", "proto", "ts"].map(|column| source_column("flows", column)); + let flows = ["src", "dst", "proto", "ts"] + .map(|column| ColumnLineage::SourceColumn(table_column("flows", 0, column))); assert_eq!(lineage(&memo, g1), flows); assert_eq!(lineage(&memo, g2), flows); // Each of the aggregate, the state and the readout computes its own @@ -148,7 +120,10 @@ fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { for group in [g3, g4, g5] { assert_eq!( lineage(&memo, group), - [source_column("flows", "src"), computed(group, 1)] + [ + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + computed(group, 1) + ] ); } // The state's selection names the filtered column by its lineage. @@ -158,7 +133,7 @@ fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { .as_ref() .expect("a summary state"); assert_eq!( - coverage.selection()[0].columns()[&column_of("flows", "proto")], + coverage.selection()[0].columns()[&table_column("flows", 0, "proto")], Constraint::In(vec![ScalarValue::Utf8("tcp".into())]) ); @@ -212,9 +187,9 @@ fn a_rename_keeps_lineage_and_the_rollup_key_check_compares_it() { assert_eq!( lineage(&memo, renamed), [ - source_column("flows", "src"), - source_column("flows", "dst"), - source_column("flows", "region"), + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + ColumnLineage::SourceColumn(table_column("flows", 0, "dst")), + ColumnLineage::SourceColumn(table_column("flows", 0, "region")), ] ); @@ -236,7 +211,7 @@ fn a_rename_keeps_lineage_and_the_rollup_key_check_compares_it() { .as_ref() .expect("a summary state"); assert_eq!( - coverage.selection()[0].columns()[&column_of("flows", "region")], + coverage.selection()[0].columns()[&table_column("flows", 0, "region")], Constraint::In(vec![ScalarValue::Utf8("us".into())]) ); @@ -250,7 +225,10 @@ fn a_rename_keeps_lineage_and_the_rollup_key_check_compares_it() { // The regrouped state is computed by the merge, not its input's. assert_eq!( rolled_up.column_lineage, - [source_column("flows", "src"), computed(rolled_up_id, 1)] + [ + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + computed(rolled_up_id, 1) + ] ); assert!(rolled_up.aggregate_coverage.is_some()); } @@ -284,10 +262,10 @@ fn join_columns_of_one_name_have_distinct_lineage() { assert_eq!( lineage(&memo, joined), [ - source_column("sales", "user"), - source_column("sales", "region"), - source_column("users", "id"), - source_column("users", "region"), + ColumnLineage::SourceColumn(table_column("sales", 0, "user")), + ColumnLineage::SourceColumn(table_column("sales", 0, "region")), + ColumnLineage::SourceColumn(table_column("users", 0, "id")), + ColumnLineage::SourceColumn(table_column("users", 0, "region")), ] ); @@ -308,11 +286,11 @@ fn join_columns_of_one_name_have_distinct_lineage() { }; assert_eq!( selected_column(&mut memo, 1), - [column_of("sales", "region")] + [table_column("sales", 0, "region")] ); assert_eq!( selected_column(&mut memo, 3), - [column_of("users", "region")] + [table_column("users", 0, "region")] ); } @@ -341,7 +319,10 @@ fn a_predicate_on_a_computed_column_stays_unlifted() { .unwrap(); assert_eq!( lineage(&memo, tagged_rows), - [source_column("flows", "src"), computed(tagged_rows, 1)] + [ + ColumnLineage::SourceColumn(table_column("flows", 0, "src")), + computed(tagged_rows, 1) + ] ); let tagged = memo .find_or_add_group(filter(equals(1, "x"), tagged_rows)) @@ -361,21 +342,7 @@ fn a_predicate_on_a_computed_column_stays_unlifted() { /// `Scan latency`: `[ts, value, job]`, `ts` its time index. fn latency_series() -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, false), - ], - 0, - vec![], - ), - }) + memo_fixtures::scan("latency", &[("job", DataType::Utf8)], vec![]) } /// A chain of `SummaryAgg[KLL](by job; value)` windows: `[window_end, job, @@ -399,7 +366,7 @@ fn a_chain_computes_window_end_and_state_and_its_window_selections_pass_lineage_ .unwrap(); let chain_lineage = [ computed(chain, 0), - latency_column("job"), + ColumnLineage::SourceColumn(series_column("latency", 0, "job")), computed(chain, 2), ]; assert_eq!(lineage(&memo, chain), chain_lineage); @@ -419,7 +386,10 @@ fn a_chain_computes_window_end_and_state_and_its_window_selections_pass_lineage_ .unwrap(); assert_eq!( lineage(&memo, merged), - [latency_column("job"), computed(merged, 1)] + [ + ColumnLineage::SourceColumn(series_column("latency", 0, "job")), + computed(merged, 1) + ] ); } @@ -432,7 +402,8 @@ fn a_time_shift_at_end_or_start_above_a_selector_passes_its_lineage_through() { let mut memo = memo(); let series = memo.find_or_add_group(latency_series()).unwrap(); let selector = memo.find_or_add_group(range(300_000, series)).unwrap(); - let series_lineage = ["ts", "value", "job"].map(latency_column); + let series_lineage = ["ts", "value", "job"] + .map(|column| ColumnLineage::SourceColumn(series_column("latency", 0, column))); for at in [AtModifier::End, AtModifier::Start] { let pinned = memo .find_or_add_group(shift(0, Some(at), selector)) @@ -525,15 +496,15 @@ fn a_self_join_names_its_two_scans_apart() { (1, "dst"), (1, "proto") ] - .map(|(scan, column)| ColumnLineage::SourceColumn(column_of_scan("flows", scan, column))) + .map(|(scan, column)| ColumnLineage::SourceColumn(table_column("flows", scan, column))) ); - let left_proto = [column_of_scan("flows", 0, "proto")]; + let left_proto = [table_column("flows", 0, "proto")]; assert_eq!(self_join_selection(&mut memo, "a", 2), left_proto); assert_eq!(self_join_selection(&mut memo, "x", 2), left_proto); assert_eq!( self_join_selection(&mut memo, "a", 5), - [column_of_scan("flows", 1, "proto")] + [table_column("flows", 1, "proto")] ); } @@ -555,22 +526,11 @@ fn match_grouping_memo() -> Memo { /// `Scan metric`: `[ts, value, *labels]`, `ts` its time index. fn series(metric: &str, labels: &[&str]) -> Operator { - let mut fields = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ]; - fields.extend( - labels - .iter() - .map(|label| Field::plain(*label, DataType::Utf8, false)), - ); - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index(fields, 0, vec![]), - }) + let labels: Vec<(&str, DataType)> = labels + .iter() + .map(|&label| (label, DataType::Utf8)) + .collect(); + memo_fixtures::scan(metric, &labels, vec![]) } /// `lhs * on(job) group_(team) rhs`. @@ -608,16 +568,6 @@ fn lineage_and_sign_of(memo: &Memo, group: MemoGroupId, name: &str) -> (ColumnLi ) } -fn series_column(metric: &str, column: &str) -> ColumnLineage { - ColumnLineage::SourceColumn(SourceColumn { - source: Source::TimeSeries { - metric: metric.into(), - }, - scan_ordinal: 0, - column: column.into(), - }) -} - /// `a * on(job) group_left(team) b`: `team`, which only `b` has, takes /// `b.team`'s lineage and sign (the "one" side's). #[test] @@ -632,7 +582,10 @@ fn a_group_left_label_comes_from_the_right_side() { .unwrap(); assert_eq!( lineage_and_sign_of(&memo, product, "team"), - (series_column("b", "team"), ValueSign::NonPositive) + ( + ColumnLineage::SourceColumn(series_column("b", 0, "team")), + ValueSign::NonPositive + ) ); } @@ -655,7 +608,10 @@ fn a_group_left_label_overwrites_the_left_sides_column_of_that_name() { assert_eq!(schema.fields.len(), 4, "no second `team` column"); assert_eq!( lineage_and_sign_of(&memo, product, "team"), - (series_column("b", "team"), ValueSign::NonPositive) + ( + ColumnLineage::SourceColumn(series_column("b", 0, "team")), + ValueSign::NonPositive + ) ); } @@ -675,7 +631,10 @@ fn a_group_right_label_comes_from_the_left_side() { .unwrap(); assert_eq!( lineage_and_sign_of(&memo, product, "team"), - (series_column("a", "team"), ValueSign::NonNegative) + ( + ColumnLineage::SourceColumn(series_column("a", 0, "team")), + ValueSign::NonNegative + ) ); } @@ -712,7 +671,7 @@ fn a_same_source_binary_op_keeps_its_labels_scan() { .unwrap(); assert_eq!( lineage_and_sign_of(&memo, ratio, "job").0, - series_column("x", "job") + ColumnLineage::SourceColumn(series_column("x", 0, "job")) ); } @@ -731,7 +690,41 @@ fn a_same_source_group_right_takes_the_right_sides_labels() { for label in ["job", "team"] { assert_eq!( lineage_and_sign_of(&memo, product, label).0, - series_column("x", label) + ColumnLineage::SourceColumn(series_column("x", 0, label)) ); } } + +/// `a or b` with only `b` having `team`: on `a`'s rows `team` is null, so it +/// is computed by the `or` with no sign, not `b.team` (before, it copied +/// `b.team` and `b`'s sign). +#[test] +fn an_or_label_only_the_right_side_has_is_computed() { + let mut memo = match_grouping_memo(); + let a = memo.find_or_add_group(series("a", &["job"])).unwrap(); + let b = memo + .find_or_add_group(series("b", &["job", "team"])) + .unwrap(); + let union = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Set(PromQLVectorSetOpKind::Or), + vector_match: None, + }, + return_bool: false, + lhs: a, + rhs: b, + })) + .unwrap(); + let team = memo.groups()[union.0 as usize] + .group_properties + .schema + .column_id("team") + .expect("`or` adds the right side's labels"); + assert_eq!( + lineage_and_sign_of(&memo, union, "team"), + (computed(union, team), ValueSign::Unknown) + ); +} diff --git a/crates/types/tests/memo_aggregate_coverage.rs b/crates/types/tests/memo_aggregate_coverage.rs index b5a80d3fb..5c96ed40b 100644 --- a/crates/types/tests/memo_aggregate_coverage.rs +++ b/crates/types/tests/memo_aggregate_coverage.rs @@ -10,7 +10,7 @@ //! ``` use std::ops::Bound; -use asap_types::ir::operator::arguments::{AtModifier, Reduction, Source, SourceColumn}; +use asap_types::ir::operator::arguments::{AtModifier, Reduction, SourceColumn}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::selection::{Constraint, IntervalConstraintLimit}; @@ -24,7 +24,7 @@ use asap_types::stages::{ OriginOfCandidateForMemoGroup, }; use asap_types::summary_type::{ExactKind, MergeInputRule, SketchAlgorithm, SummaryUpdate}; -use asap_types::test_support::memo_fixtures::{filter, range, scan, shift, state}; +use asap_types::test_support::memo_fixtures::{filter, range, scan, series_column, shift, state}; use asap_types::workload::DataWorkload; const TS: usize = 0; @@ -146,18 +146,7 @@ fn last_minute() -> Constraint { } fn ts() -> SourceColumn { - latency_column("ts") -} - -/// The column `column` of the series `latency`. -fn latency_column(column: &str) -> SourceColumn { - SourceColumn { - source: Source::TimeSeries { - metric: "latency".into(), - }, - scan_ordinal: 0, - column: column.into(), - } + series_column("latency", 0, "ts") } /// The constraint on the timestamp column of `group`'s one selection box. @@ -184,7 +173,10 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; - assert_eq!(selection.columns()[&latency_column("job")], jobs(&["api"])); + assert_eq!( + selection.columns()[&series_column("latency", 0, "job")], + jobs(&["api"]) + ); assert_eq!(selection.columns()[&ts()], last_minute()); } @@ -207,7 +199,7 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { panic!("one box: {coverage:?}") }; assert_eq!( - selection.columns()[&latency_column("job")], + selection.columns()[&series_column("latency", 0, "job")], jobs(&["api", "web"]) ); } @@ -251,7 +243,7 @@ fn a_filter_that_is_no_selection_stays_unlifted() { ] ); assert_eq!( - coverage_of_g10.selection()[0].columns()[&latency_column("job")], + coverage_of_g10.selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api"]) ); assert!(matches!( @@ -304,7 +296,7 @@ fn states_whose_definitions_read_the_same_subquery_merge() { assert_eq!(definition, coverage(&memo, web).definition()); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&latency_column("job")], + coverage(&memo, merged).selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api", "web"]) ); } @@ -324,7 +316,7 @@ fn scan_predicates_lift_and_their_states_merge() { memo.groups()[series.0 as usize] .group_properties .column_lineage[JOB], - ColumnLineage::SourceColumn(latency_column("job")) + ColumnLineage::SourceColumn(series_column("latency", 0, "job")) ); let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); memo.find_or_add_group(kll(last_minute)).unwrap() @@ -338,12 +330,12 @@ fn scan_predicates_lift_and_their_states_merge() { vec![kll_unlifted(), latency(vec![]).map_children(|_| None)] ); assert_eq!( - coverage(&memo, api).selection()[0].columns()[&latency_column("job")], + coverage(&memo, api).selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api"]) ); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&latency_column("job")], + coverage(&memo, merged).selection()[0].columns()[&series_column("latency", 0, "job")], jobs(&["api", "web"]) ); } @@ -507,7 +499,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { } ); assert_eq!( - coverage(&memo, g3).selection()[0].columns()[&latency_column("value")], + coverage(&memo, g3).selection()[0].columns()[&series_column("latency", 0, "value")], Constraint::Interval { lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( 10.0 @@ -587,7 +579,10 @@ fn an_exact_aggregate_has_coverage() { let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; - assert_eq!(selection.columns()[&latency_column("job")], jobs(&["api"])); + assert_eq!( + selection.columns()[&series_column("latency", 0, "job")], + jobs(&["api"]) + ); assert_eq!(selection.columns()[&ts()], last_minute()); } diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index cb822d7a6..1c20e298d 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -105,7 +105,7 @@ enum NonASAPOp { Values { rows: Vec>, schema: Schema }, Filter { child: Rc, pred: Predicate }, Project { - child: Rc, cols: Vec, qualifier: Option, + child: Rc, cols: Vec, }, Aggregate { child: Rc, reduction: Reduction, measures: Vec, diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index f3346279b..1eecb5935 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -75,7 +75,6 @@ pub struct Field { pub name: String, pub dtype: FieldDataType, // Plain(DataType) | ExactAggregate(..) | Sketch(..) | SummarySample(..) | Wavelet(..) | StatModel(..) pub nullable: bool, - pub table: Option, // SQL table/alias qualifier; None for PromQL labels } ``` @@ -156,9 +155,10 @@ gated by `NonASAPOp::is_shareable_when_equal` for non-ASAP producers. ## Fields and column references -`Schema` holds `Field` metadata (name, type, nullability, qualifier) and no -values; an unresolved `ColumnRef` resolves to a positional `ColumnId` within one -schema. The design, including how the same position selects a runtime value, is +`Schema` holds `Field` metadata (name, type, nullability) and no values; an +unresolved `ColumnRef` resolves to a positional `ColumnId` within one schema. A +front end binds a qualified `table.column` with its own `Scope` while it +lowers; the IR keeps no qualifier. The design, including how the same position selects a runtime value, is in [Schema and physical data for ASAP primitives](../design_docs/proposals/asap-primitive-schema.md#3-proposed-schema-design). ## Node index @@ -567,7 +567,6 @@ SELECT srcip, dstip FROM packets **Fields:** - `cols` — the output column list (expression + optional alias per column). -- `qualifier` — a table alias re-qualifying every output column, for a derived table; `None` for an ordinary `SELECT` list. - `child` — the input being projected. ### BinaryOp From e470e59be22d0f006cdb551385ac5a2639ba5d88 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:19:48 +0000 Subject: [PATCH 5/5] docs(operator-sharing): columns name their source by lineage, not schema qualifiers --- docs/design_docs/proposals/operator-sharing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index 1c20e298d..52db97989 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -145,8 +145,9 @@ Ordinary payload fields have these roles: `measures` describes the requested aggregates. Grouping is distinct from ordering or limiting within groups, represented by `partition_by`. - Join/set kinds, vector matching, window frames and time selections preserve - source-language semantics. Output names, qualifiers and proven uniqueness also - survive optimization. + source-language semantics. Output names and proven uniqueness also survive + optimization; a column's source is its lineage (a memo-group property), not a + table qualifier on the schema. `ASAPOp` describes state construction, state operations and evaluation separately. `FieldDataType` (§2.1) types every output field; state-producing operations use its