Skip to content

Commit 01413dc

Browse files
committed
fix: read event-keyed warehouse history on Snowflake and report metadata read errors
History reads appended event_id to projections and orderings that already contained it, which Snowflake rejects as an ambiguous column. Select and order by it once, keeping each kind's fixed-length cursor. Failed metadata reads and warehouse connections now report the kind and the redacted driver error.
1 parent d15ee25 commit 01413dc

14 files changed

Lines changed: 295 additions & 47 deletions

File tree

‎crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1232,7 +1232,13 @@ pub(crate) async fn read_metadata_page(
12321232
Some(reader) => reader,
12331233
None => match open_location_runtime(options, script, Vec::new()) {
12341234
Ok(reader) => reader,
1235-
Err(_) => return Ok(Some((state, None))),
1235+
Err(error) => {
1236+
drop(write_output(&[(
1237+
Stream::Stderr,
1238+
&format!("Warehouse metadata reader did not start: {error}\n"),
1239+
)]));
1240+
return Ok(Some((state, None)));
1241+
}
12361242
},
12371243
};
12381244
tokio::pin!(stopped);
@@ -1254,11 +1260,23 @@ pub(crate) async fn read_metadata_page(
12541260
state.reader = Some(reader);
12551261
Ok(Some((state, None)))
12561262
}
1257-
Ok((reader, Err(_))) => {
1263+
Ok((reader, Err(error))) => {
1264+
drop(write_output(&[(
1265+
Stream::Stderr,
1266+
&format!("Warehouse metadata reader failed: {error}\n"),
1267+
)]));
12581268
stop_location_runtime(Some(reader)).await?;
12591269
Ok(Some((state, None)))
12601270
}
1261-
Err(_) => Ok(Some((state, None))),
1271+
Err(_) => {
1272+
drop(write_output(&[(
1273+
Stream::Stderr,
1274+
&format!(
1275+
"Warehouse metadata read did not answer within {METADATA_POLL_SECONDS}s.\n"
1276+
),
1277+
)]));
1278+
Ok(Some((state, None)))
1279+
}
12621280
},
12631281
}
12641282
}

‎crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs‎

Lines changed: 39 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22
33
use std::io::{self, Write};
44

5+
use pyo3::PyErr;
56
use pyo3::exceptions::PyValueError;
67
use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python};
78
use pyo3::types::{PyDict, PyDictMethods};
9+
use sqlbuild_core::redaction::main::redact_error_text::redact_error_text;
810
use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest};
911
use sqlbuild_orchestration::warehouse_metadata::models::{
1012
WarehouseHistoryKind, WarehouseHistoryRead,
@@ -55,9 +57,9 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul
5557
}
5658
};
5759
serde_json::to_string(&record)
58-
} else if WarehouseHistoryKind::ALL
60+
} else if let Some(kind) = WarehouseHistoryKind::ALL
5961
.iter()
60-
.any(|kind| request.get(kind.as_str()).is_some())
62+
.find(|kind| request.get(kind.as_str()).is_some())
6163
{
6264
let metadata: WarehouseHistoryRead =
6365
serde_json::from_value(request).map_err(|_| {
@@ -73,7 +75,10 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul
7375
if !busy {
7476
write_stderr(
7577
py,
76-
"Warehouse connection for metadata reads failed.\n",
78+
&format!(
79+
"Warehouse connection for metadata reads failed: {}\n",
80+
failure_reason(&error)
81+
),
7782
)?;
7883
} else if !reported_busy {
7984
write_stderr(
@@ -86,16 +91,13 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul
8691
};
8792
}
8893
let page = match &session {
89-
Some(opened) => {
90-
match read_history(py, (opened, stream.take()), &metadata) {
91-
Ok((page, open)) => {
92-
stream = open;
93-
Some(page)
94-
}
95-
Err(_) => None,
96-
}
97-
}
98-
None => None,
94+
Some(opened) => read_history(py, (opened, stream.take()), &metadata)
95+
.map(|(page, open)| {
96+
stream = open;
97+
page
98+
})
99+
.map_err(|error| format!(": {}", failure_reason(&error))),
100+
None => Err(String::new()),
99101
};
100102
if session.as_ref().is_some_and(|opened| opened.local)
101103
&& let Some(opened) = session.take()
@@ -106,16 +108,23 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul
106108
if busy {
107109
serde_json::to_string(&serde_json::json!({ "busy": true }))
108110
} else {
109-
if page.is_none() {
110-
if let Some(opened) = session.take() {
111-
drop(close_history_session(opened, None));
111+
let delivered = match &page {
112+
Ok(page) => Some(page),
113+
Err(reason) => {
114+
if let Some(opened) = session.take() {
115+
drop(close_history_session(opened, None));
116+
}
117+
write_stderr(
118+
py,
119+
&format!(
120+
"Warehouse metadata read ({}) failed{reason}. No page delivered.\n",
121+
kind.as_str()
122+
),
123+
)?;
124+
None
112125
}
113-
write_stderr(
114-
py,
115-
"Warehouse metadata read failed. No page delivered.\n",
116-
)?;
117-
}
118-
serde_json::to_string(&page)
126+
};
127+
serde_json::to_string(&delivered)
119128
}
120129
} else {
121130
let evaluation: SensorEvaluationRequest = serde_json::from_value(request)
@@ -162,3 +171,11 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul
162171
result
163172
})
164173
}
174+
175+
/// The driver's class and message on one line, redacted as warehouse error text.
176+
fn failure_reason(error: &PyErr) -> String {
177+
redact_error_text(&error.to_string())
178+
.split_whitespace()
179+
.collect::<Vec<_>>()
180+
.join(" ")
181+
}

‎crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use sqlbuild_state::freshness::constants::{
2929
};
3030
use sqlbuild_state::observations::constants::OBSERVATION_COLUMN_TYPES;
3131
use sqlbuild_state::resource_costs::constants::RESOURCE_COST_COLUMN_TYPES;
32+
use sqlbuild_state::state_tables::main::event_id_index::event_id_index;
3233
use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql;
3334
use sqlbuild_state::state_tables::models::HistoryRead;
3435

@@ -445,7 +446,7 @@ fn read_page<'py>(
445446
.then_some(index)
446447
})
447448
.collect();
448-
cursor_indexes.push(columns.len());
449+
cursor_indexes.push(event_id_index(columns));
449450
let mut literals = vec![None; key_columns.unwrap_or(columns).len() + 1];
450451
if let Some(after) = after {
451452
let classes = SqlValueClasses::load(py)?;
@@ -771,7 +772,7 @@ fn decode_record(
771772
let mut value = json!({
772773
"state_database": location.database,
773774
"state_schema": location.schema,
774-
"event_id": row.get_item(columns.len())?.extract::<Option<String>>()?,
775+
"event_id": row.get_item(event_id_index(columns))?.extract::<Option<String>>()?,
775776
});
776777
for name in columns {
777778
let name = match *name {

‎crates/sqlbuild-state/src/state_tables/constants.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,5 @@
33
pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver";
44
/// Cursor key of a table without event IDs: always NULL, so it never orders or binds a value.
55
pub(crate) const MISSING_EVENT_ID_KEY: &str = "NULL";
6+
/// The column every event table identifies its rows by.
7+
pub(crate) const EVENT_ID_COLUMN: &str = "event_id";
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
//! Where a history read's row carries its event ID.
2+
3+
use crate::state_tables::constants::EVENT_ID_COLUMN;
4+
5+
/// A selected `event_id` column, else the one the history read appends after the columns.
6+
#[must_use]
7+
pub fn event_id_index(columns: &[&str]) -> usize {
8+
columns
9+
.iter()
10+
.position(|column| *column == EVENT_ID_COLUMN)
11+
.unwrap_or(columns.len())
12+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
pub mod create_table_sql;
2+
pub mod event_id_index;
23
pub mod read_history_sql;
34
pub mod values_insert_sql;

‎crates/sqlbuild-state/src/state_tables/main/read_history_sql.rs‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
//! Keyset pages preserve distinct observations without rereading or ranking all preceding rows.
22
3-
use crate::state_tables::constants::{MISSING_EVENT_ID_KEY, SQLSERVER_ADAPTER};
3+
use crate::state_tables::constants::{EVENT_ID_COLUMN, MISSING_EVENT_ID_KEY, SQLSERVER_ADAPTER};
44
use crate::state_tables::models::HistoryRead;
55

6-
/// Cursor values are already rendered by the adapter; `None` sorts before every SQL value.
6+
/// Rendered cursor values, `None` first; a trailing event ID repeating an `event_id` key is ignored.
77
pub fn read_history_sql(
88
qualified_name: &str,
99
after: Option<&[Option<String>]>,
@@ -19,17 +19,21 @@ pub fn read_history_sql(
1919
limit,
2020
} = read;
2121
let mut projection = columns.join(", ");
22-
projection.push_str(if has_event_id {
23-
", event_id"
24-
} else {
25-
", NULL AS event_id"
26-
});
27-
let mut columns = key_columns.unwrap_or(columns).to_vec();
28-
columns.push(if has_event_id {
29-
"event_id"
30-
} else {
31-
MISSING_EVENT_ID_KEY
32-
});
22+
if !columns.contains(&EVENT_ID_COLUMN) {
23+
projection.push_str(if has_event_id {
24+
", event_id"
25+
} else {
26+
", NULL AS event_id"
27+
});
28+
}
29+
let mut columns: Vec<&str> = key_columns.unwrap_or(columns).to_vec();
30+
if !columns.contains(&EVENT_ID_COLUMN) {
31+
columns.push(if has_event_id {
32+
EVENT_ID_COLUMN
33+
} else {
34+
MISSING_EVENT_ID_KEY
35+
});
36+
}
3337
let ordered = columns
3438
.iter()
3539
.map(|column| {

‎crates/sqlbuild-state/src/state_tables/tests/test_sql.rs‎

Lines changed: 127 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1+
use crate::events::constants::{
2+
COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, NODE_RESULT_COLUMNS, OLD_NAME_VIEW_COLUMNS,
3+
};
14
use crate::state_tables::main::create_table_sql::create_table_sql;
25
use crate::state_tables::main::read_history_sql::read_history_sql;
36
use crate::state_tables::main::values_insert_sql::values_insert_sql;
47
use crate::state_tables::models::HistoryRead;
58
use crate::state_tables::tests::helpers::column;
69
use crate::state_tables::tests::test_types::{
7-
CreateTableTestCase, HistoryReadTestCase, ValuesInsertTestCase,
10+
CreateTableTestCase, EventKeyTestCase, HistoryReadTestCase, ValuesInsertTestCase,
811
};
912

1013
#[test]
@@ -110,3 +113,126 @@ fn given_a_table_without_event_ids_when_paging_history_then_the_missing_key_neve
110113
);
111114
}
112115
}
116+
117+
#[test]
118+
fn given_event_tables_keyed_by_event_id_when_paging_history_then_each_column_and_key_appears_once()
119+
{
120+
let test_cases = [
121+
EventKeyTestCase {
122+
description: "migrations",
123+
columns: &MIGRATION_COLUMNS,
124+
key_columns: &["event_id", "created_at"],
125+
timestamp_column: "created_at",
126+
expected_ordering_keys: &["event_id", "created_at"],
127+
},
128+
EventKeyTestCase {
129+
description: "column migrations",
130+
columns: &COLUMN_MIGRATION_COLUMNS,
131+
key_columns: &["event_id", "created_at"],
132+
timestamp_column: "created_at",
133+
expected_ordering_keys: &["event_id", "created_at"],
134+
},
135+
EventKeyTestCase {
136+
description: "old-name views",
137+
columns: &OLD_NAME_VIEW_COLUMNS,
138+
key_columns: &["event_id", "created_at"],
139+
timestamp_column: "created_at",
140+
expected_ordering_keys: &["event_id", "created_at"],
141+
},
142+
EventKeyTestCase {
143+
description: "microbatches",
144+
columns: &["event_id", "record_type", "scope_kind", "created_at"],
145+
key_columns: &["event_id", "created_at"],
146+
timestamp_column: "created_at",
147+
expected_ordering_keys: &["event_id", "created_at"],
148+
},
149+
EventKeyTestCase {
150+
description: "janitor events",
151+
columns: &["event_id", "schema_version", "event_type", "occurred_at"],
152+
key_columns: &["event_id", "occurred_at"],
153+
timestamp_column: "occurred_at",
154+
expected_ordering_keys: &["event_id", "occurred_at"],
155+
},
156+
EventKeyTestCase {
157+
description: "node results, which carry no event ID column",
158+
columns: &NODE_RESULT_COLUMNS,
159+
key_columns: &[
160+
"node_type",
161+
"node_name",
162+
"target_database",
163+
"target_schema",
164+
"target_name",
165+
"run_id",
166+
"status",
167+
"ts",
168+
],
169+
timestamp_column: "ts",
170+
expected_ordering_keys: &[
171+
"node_type",
172+
"node_name",
173+
"target_database",
174+
"target_schema",
175+
"target_name",
176+
"run_id",
177+
"status",
178+
"ts",
179+
"event_id",
180+
],
181+
},
182+
];
183+
let adapters = [
184+
"snowflake",
185+
"duckdb",
186+
"motherduck",
187+
"sqlite",
188+
"postgres",
189+
"bigquery",
190+
"databricks",
191+
"sqlserver",
192+
"clickhouse",
193+
];
194+
for test_case in test_cases {
195+
for adapter in adapters {
196+
let protocol_cursor = vec![Some("'x'".to_owned()); test_case.key_columns.len() + 1];
197+
let sql = read_history_sql(
198+
"s.t",
199+
Some(&protocol_cursor),
200+
HistoryRead {
201+
columns: test_case.columns,
202+
key_columns: Some(test_case.key_columns),
203+
timestamp_column: test_case.timestamp_column,
204+
observed_since: None,
205+
adapter,
206+
has_event_id: true,
207+
limit: 5,
208+
},
209+
)
210+
.expect("supported adapter");
211+
let head = sql.strip_prefix("SELECT ").expect("select");
212+
let head = head.strip_prefix("TOP 5 ").unwrap_or(head);
213+
let (projection, rest) = head.split_once(" FROM ").expect("from");
214+
let mut selected: Vec<&str> = projection.split(", ").collect();
215+
selected.sort_unstable();
216+
selected.dedup();
217+
assert_eq!(
218+
selected.len(),
219+
test_case.columns.len() + usize::from(!test_case.columns.contains(&"event_id")),
220+
"{} on {adapter}: each column selected once in {sql}",
221+
test_case.description
222+
);
223+
let ordering_keys: Vec<&str> = rest
224+
.split_once(" ORDER BY ")
225+
.expect("order")
226+
.1
227+
.split("CASE WHEN ")
228+
.skip(1)
229+
.filter_map(|item| item.split_once(" IS NULL").map(|(key, _)| key))
230+
.collect();
231+
assert_eq!(
232+
ordering_keys, test_case.expected_ordering_keys,
233+
"{} on {adapter}: {sql}",
234+
test_case.description
235+
);
236+
}
237+
}
238+
}

‎crates/sqlbuild-state/src/state_tables/tests/test_types.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,11 @@ pub(crate) struct HistoryReadTestCase {
1919
pub(crate) after: Option<Vec<Option<String>>>,
2020
pub(crate) expected_sql: &'static str,
2121
}
22+
23+
pub(crate) struct EventKeyTestCase {
24+
pub(crate) description: &'static str,
25+
pub(crate) columns: &'static [&'static str],
26+
pub(crate) key_columns: &'static [&'static str],
27+
pub(crate) timestamp_column: &'static str,
28+
pub(crate) expected_ordering_keys: &'static [&'static str],
29+
}

0 commit comments

Comments
 (0)