Skip to content

Commit 4698f18

Browse files
committed
wip: keep expression ranges in the snapshot columns document
Ranges move out of the node definition into the per-snapshot columns document, so they never change a node hash or record a node change; the node SQL read ignores unknown per-column keys and drops malformed or misplaced ranges.
1 parent 28645e8 commit 4698f18

10 files changed

Lines changed: 115 additions & 32 deletions

File tree

‎benchmarks/duckshop/edge_cases/orchestration/node_sql.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,13 @@
44
"python_source": "def order_amount():\n return 23\n",
55
"function_sql": "43",
66
"updated_sql": "SELECT 37 AS amount -- Updated orders 🧾\n",
7-
"expression_ranges": {"amount": [7, 19]}
7+
"columns_document": {
8+
"amount": {"expression_range": [7, 19], "described_as": "a later per-column key readers ignore"},
9+
"past_the_end": {"expression_range": [7, 4096]},
10+
"split_character": {"expression_range": [7, 32]},
11+
"reversed": {"expression_range": [19, 7]},
12+
"no_range": {}
13+
},
14+
"expression_ranges": {"amount": [7, 19]},
15+
"updated_expression_ranges": {"amount": [7, 19], "split_character": [7, 32]}
816
}

‎crates/sqlbuild-orchestration/src/snapshots/_helpers/preparation.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,11 @@ pub(crate) fn prepare_snapshot(input: &SnapshotInput) -> Result<PreparedSnapshot
4848
})
4949
}
5050

51+
/// Documents are per-snapshot metadata outside node identity: changing one never changes a node
52+
/// hash or records a node change. `columns` holds per-column metadata by node key,
53+
/// `{node_key: {column: {"expression_range": [start, end]}}}`, where `expression_range` is the
54+
/// column's defining expression as `[start, end)` UTF-8 byte offsets of the node's compiled SQL.
55+
/// Readers ignore per-column keys they do not know and treat a missing or malformed range as none.
5156
fn prepare_documents(
5257
input: &SnapshotInput,
5358
) -> Result<BTreeMap<String, (String, Vec<u8>)>, ControlError> {

‎crates/sqlbuild-orchestration/src/snapshots/main/read_person_node_sql.rs‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
//! Live SQL permission, current publication and sensitive-read audit share one writer fence.
22
3-
use serde_json::json;
3+
use std::collections::BTreeMap;
4+
5+
use serde_json::{Value, json};
46
use uuid::Uuid;
57

68
use crate::authorization::main::begin_authorized::begin_authorized;
79
use crate::credentials::main::audit::append_audit::append_audit;
810
use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope};
911
use crate::snapshots::_helpers::reading::attach_sources;
1012
use crate::snapshots::main::read_current_record::read_current_record;
13+
use crate::snapshots::main::read_snapshot_documents::read_snapshot_documents;
1114
use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes;
1215
use crate::snapshots::models::NodeSql;
1316
use crate::store::errors::ControlError;
@@ -40,12 +43,16 @@ pub async fn read_person_node_sql(
4043
WHERE s.snapshot_id=$1 AND s.node_key=$2 AND s.kind IN ('raw_sql','compiled_sql')",
4144
vec![Parameter::Id(record.snapshot_id), Parameter::Text(node_key.into())]).await?;
4245
if let Some(node) = attach_sources(nodes, rows)?.remove(node_key) {
43-
let expression_ranges = node
44-
.definition
45-
.get("expression_ranges")
46-
.cloned()
47-
.and_then(|ranges| serde_json::from_value(ranges).ok())
48-
.unwrap_or_default();
46+
let documents;
47+
(transaction, documents) =
48+
read_snapshot_documents(transaction, record.snapshot_id, Some(&["columns"]))
49+
.await?;
50+
let expression_ranges = expression_ranges(
51+
documents
52+
.get("columns")
53+
.and_then(|columns| columns.get(node_key)),
54+
node.compiled_sql.as_deref().unwrap_or_default(),
55+
);
4956
result = Some(NodeSql {
5057
snapshot_id: record.snapshot_id,
5158
node_key: node_key.into(),
@@ -66,3 +73,26 @@ pub async fn read_person_node_sql(
6673
commit(transaction).await?;
6774
Ok(result)
6875
}
76+
77+
/// Each column's expression range from the node's `columns` document entry; a column whose
78+
/// range is missing, malformed or off the compiled SQL's character boundaries has none.
79+
fn expression_ranges(columns: Option<&Value>, compiled_sql: &str) -> BTreeMap<String, [usize; 2]> {
80+
let Some(columns) = columns.and_then(Value::as_object) else {
81+
return BTreeMap::new();
82+
};
83+
columns
84+
.iter()
85+
.filter_map(|(column, metadata)| {
86+
let range = metadata.get("expression_range")?.as_array()?;
87+
let [start, end] = range.as_slice() else {
88+
return None;
89+
};
90+
let start = usize::try_from(start.as_u64()?).ok()?;
91+
let end = usize::try_from(end.as_u64()?).ok()?;
92+
compiled_sql
93+
.get(start..end)
94+
.filter(|text| !text.is_empty())
95+
.map(|_| (column.clone(), [start, end]))
96+
})
97+
.collect()
98+
}

‎crates/sqlbuild-orchestration/tests/api/node_sql/helpers.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,6 @@ pub(crate) async fn node_sql_boundaries(
125125
assert_eq!(response.status(), StatusCode::OK);
126126
assert_eq!(
127127
response.json::<Value>().await.expect("latest SQL"),
128-
json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"],"expression_ranges":data["expression_ranges"]})
128+
json!({"snapshot_id":current.snapshot_id,"node_key":"orders","raw_sql":data["raw_sql"],"compiled_sql":data["updated_sql"],"expression_ranges":data["updated_expression_ranges"]})
129129
);
130130
}

‎crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,13 +115,16 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_
115115
.expect("compiled SQL")
116116
.into(),
117117
);
118-
orders.definition["expression_ranges"] = sql_input["expression_ranges"].clone();
119118
orders.python_source = Some(
120119
sql_input["python_source"]
121120
.as_str()
122121
.expect("Python source")
123122
.into(),
124123
);
124+
input.documents.insert(
125+
"columns".into(),
126+
json!({"orders": sql_input["columns_document"].clone()}),
127+
);
125128
input.nodes.insert(
126129
"udf:orders".into(),
127130
SnapshotNode {

‎crates/sqlbuild-orchestration/tests/snapshots/helpers.rs‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
use std::collections::BTreeMap;
44

5+
use crate::snapshots::_helpers::preparation::prepare_snapshot;
56
use crate::snapshots::main::current_snapshot::current_snapshot;
67
use crate::snapshots::main::publish_snapshot::publish_snapshot;
78
use crate::snapshots::models::{SnapshotInput, SnapshotNode};
@@ -113,7 +114,30 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 {
113114
Err(ControlError::StaleGeneration)
114115
));
115116

117+
// Column metadata lives outside node identity: new expression ranges alone keep every node
118+
// hash and record no node change.
119+
let before = input.clone();
116120
input.generation = 3;
121+
input.documents.insert(
122+
"columns".into(),
123+
json!({"orders": {"customer_id": {"expression_range": [7, 18]}}}),
124+
);
125+
assert_eq!(
126+
prepare_snapshot(&input).expect("prepare ranged").hashes,
127+
prepare_snapshot(&before).expect("prepare unranged").hashes
128+
);
129+
let ranged = publish_snapshot(store, &input)
130+
.await
131+
.expect("publish expression ranges only");
132+
assert_ne!(ranged.manifest_digest, unchanged.manifest_digest);
133+
let loaded = current_snapshot(store, location, "development")
134+
.await
135+
.expect("read ranged snapshot")
136+
.expect("ranged snapshot exists");
137+
assert!(loaded.changes.is_empty());
138+
assert_eq!(loaded.documents, input.documents);
139+
140+
input.generation = 4;
117141
input
118142
.nodes
119143
.get_mut("orders")
@@ -146,7 +170,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 {
146170
.expect("development target exists")
147171
.record
148172
.generation,
149-
3
173+
4
150174
);
151175
assert_eq!(
152176
current_snapshot(store, location, "testing")
@@ -159,7 +183,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 {
159183
);
160184

161185
input.target = "development".into();
162-
input.generation = 4;
186+
input.generation = 5;
163187
input.nodes.remove("orders");
164188
publish_snapshot(store, &input)
165189
.await
@@ -175,8 +199,8 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 {
175199

176200
// The newer compiler wins regardless of which publisher takes the transaction lock first.
177201
let mut older = input.clone();
178-
older.generation = 5;
179-
input.generation = 6;
202+
older.generation = 6;
203+
input.generation = 7;
180204
input
181205
.nodes
182206
.get_mut("customers")
@@ -192,7 +216,7 @@ async fn exercise_snapshot_contract(store: &ControlStore) -> i64 {
192216
.await
193217
.expect("read winning snapshot")
194218
.expect("winning snapshot exists");
195-
assert_eq!(loaded.record.generation, 6);
219+
assert_eq!(loaded.record.generation, 7);
196220
assert_eq!(loaded.nodes, input.nodes);
197221
close_store(second).await;
198222
loaded.record.generation

‎crates/sqlbuild-orchestration/tests/snapshots/test_contract.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ fn given_control_store_when_snapshots_publish_then_generations_sources_and_chang
1010
SnapshotTestCase {
1111
description: "SQLite snapshot publication",
1212
backend: Backend::Sqlite,
13-
expected_generation: 6,
13+
expected_generation: 7,
1414
},
1515
SnapshotTestCase {
1616
description: "PostgreSQL snapshot publication",
1717
backend: Backend::Postgres,
18-
expected_generation: 6,
18+
expected_generation: 7,
1919
},
2020
];
2121
for test_case in test_cases {

‎crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,16 @@ use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompil
2222
use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord;
2323
use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView;
2424

25+
/// Each model's snapshot node, and the `columns` document: per node key, each column's
26+
/// expression range. The ranges stay out of the node, so they never change a node's identity.
2527
pub(crate) fn model_nodes(
2628
py: Python<'_>,
2729
project: &NativeProjectView,
2830
owner: &NativeCompiledProject,
2931
mut nodes: BTreeMap<String, SnapshotNode>,
30-
) -> PyResult<BTreeMap<String, SnapshotNode>> {
32+
) -> PyResult<(BTreeMap<String, SnapshotNode>, Map<String, Value>)> {
3133
let dialect: Option<String> = project.sql_analysis_dialect();
34+
let mut columns_document: Map<String, Value> = Map::new();
3235
for model in project.models(py).bind(py).iter() {
3336
let model = model.cast::<NativeCompiledModel>()?.get();
3437
let (name, _, _) = model.file_texts(py)?;
@@ -39,7 +42,7 @@ pub(crate) fn model_nodes(
3942
"model config cannot be represented in snapshot metadata",
4043
));
4144
}
42-
let mut definition = model.read(py, |input, _| {
45+
let (mut definition, ranges) = model.read(py, |input, _| {
4346
let record = input
4447
.compiled
4548
.as_ref()
@@ -54,7 +57,7 @@ pub(crate) fn model_nodes(
5457
json!({"name": name, "type": data_type, "nullability": nullability})
5558
})
5659
.collect();
57-
let mut definition: Value = json!({
60+
let definition: Value = json!({
5861
"name": facts.name,
5962
"path": facts.relative_path,
6063
"config": facts.config.values,
@@ -67,11 +70,11 @@ pub(crate) fn model_nodes(
6770
(Some(dialect), Some(lineage)) => expression_ranges(dialect, input, facts, lineage),
6871
_ => Map::new(),
6972
};
70-
if !ranges.is_empty() {
71-
definition["expression_ranges"] = Value::Object(ranges);
72-
}
73-
Ok(definition)
73+
Ok((definition, ranges))
7474
})?;
75+
if !ranges.is_empty() {
76+
columns_document.insert(format!("model:{}", facts.name), Value::Object(ranges));
77+
}
7578
definition["destination"] = model.read_destination(py, |value| Ok(destination(value)))?;
7679
nodes = insert_node(
7780
nodes,
@@ -85,11 +88,12 @@ pub(crate) fn model_nodes(
8588
},
8689
)?;
8790
}
88-
Ok(nodes)
91+
Ok((nodes, columns_document))
8992
}
9093

91-
/// Each output column's defining expression as a `[start, end)` byte range of the compiled SQL
92-
/// the node SQL read returns; empty when the analysis SQL cannot be rebuilt exactly.
94+
/// Each output column's defining expression as `{"expression_range": [start, end)}`, byte
95+
/// offsets in the compiled SQL the node SQL read returns; empty when the analysis SQL cannot be
96+
/// rebuilt exactly.
9397
fn expression_ranges(
9498
dialect: &str,
9599
input: &ModelInputRecord,
@@ -127,7 +131,7 @@ fn expression_ranges(
127131
if let Some((start, end)) = range {
128132
ranges
129133
.entry(output_column.clone())
130-
.or_insert_with(|| json!([start, end]));
134+
.or_insert_with(|| json!({"expression_range": [start, end]}));
131135
}
132136
}
133137
ranges

‎crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,13 @@ pub(crate) fn orchestration_snapshot<'py>(
9898
input.sqb_version = env!("CARGO_PKG_VERSION").into();
9999
input.environment_hash = environment_hash(py, &project_dir)?;
100100
let owner = project.owner().bind(py).borrow();
101-
input.nodes = model_nodes(py, &project, &owner, input.nodes)?;
101+
let (nodes, columns) = model_nodes(py, &project, &owner, input.nodes)?;
102+
input.nodes = nodes;
103+
if !columns.is_empty() {
104+
input
105+
.documents
106+
.insert("columns".into(), serde_json::Value::Object(columns));
107+
}
102108
input.nodes = resource_nodes(py, &project, &owner, input.nodes)?;
103109
drop(owner);
104110
let declarations = match declarations {

‎tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -574,12 +574,15 @@ def test_given_rewritten_model_sql_when_publishing_then_expression_ranges_slice_
574574
project, json.dumps(METADATA), tmp_path, discovered_inputs=discovered
575575
).to_json()
576576
)
577-
node: dict[str, Any] = publication["nodes"][f"model:{test_case.model_name}"]
577+
key: str = f"model:{test_case.model_name}"
578+
node: dict[str, Any] = publication["nodes"][key]
578579
assert node["compiled_sql"] != node["raw_sql"]
580+
assert "expression_range" not in json.dumps(node), "ranges stay out of node identity"
579581
compiled: bytes = node["compiled_sql"].encode()
582+
columns: dict[str, Any] = publication["documents"].get("columns", {}).get(key, {})
580583
assert {
581-
column: compiled[start:end].decode()
582-
for column, (start, end) in node["definition"].get("expression_ranges", {}).items()
584+
column: compiled[slice(*metadata["expression_range"])].decode()
585+
for column, metadata in columns.items()
583586
} == test_case.expected_texts
584587

585588

0 commit comments

Comments
 (0)