Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
4bd5fda
wip: carry the active target's settings into Python node, loader and …
kvlonge Nov 16, 2026
3ba687f
fix: stop a concurrent build when an error escapes a node
kvlonge Nov 16, 2026
4551bc3
fix: keep the first escaped error explicitly and widen the test's rac…
kvlonge Nov 16, 2026
5a79bb7
Merge remote-tracking branch 'origin/main' into fix/concurrent-worker…
kvlonge Nov 16, 2026
6877fee
refactor: build the escaped-error completion with NodeCompletion dire…
kvlonge Nov 16, 2026
7a10672
Merge remote-tracking branch 'origin/main' into feat/python-target-co…
kvlonge Nov 16, 2026
483a5e7
feat: build the active target's settings natively for node, hook and …
kvlonge Nov 16, 2026
4c58c2f
test: Python tasks, loaders and checks see the locally overridden tar…
kvlonge Nov 16, 2026
2552ea1
docs: document ctx.target_config on node, loader and hook contexts
kvlonge Nov 16, 2026
e47b4d5
refactor: make TargetSettings a native record resolved once per command
kvlonge Nov 16, 2026
78d4c1c
feat: report statement counts and compile time after sqb test and sce…
kvlonge Nov 16, 2026
672319a
Merge remote-tracking branch 'origin/main' into feat/python-target-co…
kvlonge Nov 16, 2026
2b20da2
Merge remote-tracking branch 'origin/wip/concurrent-worker-stop' into…
kvlonge Nov 16, 2026
c8c5654
Merge remote-tracking branch 'origin/wip/python-target-config' into f…
kvlonge Nov 16, 2026
0adc634
fix: keep commands working with a CTX-templated target schema; bound …
kvlonge Nov 16, 2026
918bba6
feat: pass the active target to provider setup in build, check and load
kvlonge Nov 16, 2026
afc2824
Merge remote-tracking branch 'origin/main' into feat/target-settings-…
kvlonge Nov 16, 2026
93b70ea
fix: print the sqb test results before reading statement history
kvlonge Nov 16, 2026
35a7c8a
chore: set the Python bindings fensu budgets to their measured values
kvlonge Nov 16, 2026
81367a6
Merge remote-tracking branch 'origin/main' into feat/target-settings-…
kvlonge Nov 16, 2026
10b2e05
refactor: share one JSON section helper between the since and stateme…
kvlonge Nov 16, 2026
ae781cd
test: allow table-name rows in the worker execution cases
kvlonge Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
//! The Snowflake `QUERY_HISTORY` table function, qualified by database when there is one.

use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION;

/// `INFORMATION_SCHEMA.QUERY_HISTORY`, under `database` when it is given.
pub(crate) fn query_history_function(database: Option<&str>) -> String {
match database {
None => QUERY_HISTORY_FUNCTION.to_owned(),
Some(database) => format!(
"\"{}\".{QUERY_HISTORY_FUNCTION}",
database.to_uppercase().replace('"', "\"\"")
),
}
}
1 change: 1 addition & 0 deletions crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub(crate) mod allocation;
pub(crate) mod decimal;
pub(crate) mod history_function;
pub(crate) mod query_text;
1 change: 1 addition & 0 deletions crates/sqlbuild-core/src/cost_allocation/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
pub mod allocate_run_cost;
pub mod query_history_sql;
pub mod statement_timings_sql;
Original file line number Diff line number Diff line change
@@ -1,19 +1,9 @@
//! `query_history_sql`: the Snowflake `QUERY_HISTORY` read of one end-time window.

use crate::cost_allocation::_helpers::history_function::query_history_function;
use crate::cost_allocation::_helpers::query_text::query_text_expression;
use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION;
use crate::cost_allocation::models::QueryHistoryWindow;

fn query_history_function(database: Option<&str>) -> String {
match database {
None => QUERY_HISTORY_FUNCTION.to_owned(),
Some(database) => format!(
"\"{}\".{QUERY_HISTORY_FUNCTION}",
database.to_uppercase().replace('"', "\"\"")
),
}
}

/// Query-history SQL for queries ending after `start`, and before `end` when it is bounded.
pub fn query_history_sql(window: &QueryHistoryWindow<'_>) -> String {
let end_range: String = match window.end {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
//! `statement_timings_sql`: compile and execution time of known Snowflake queries.

use crate::cost_allocation::_helpers::history_function::query_history_function;
use crate::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT;

/// One query-history read summing compile and execution milliseconds of `query_ids` that ended in the last `lookback_seconds`.
#[must_use]
pub fn statement_timings_sql(
database: Option<&str>,
query_ids: &[String],
lookback_seconds: u64,
) -> String {
let identifiers: String = query_ids
.iter()
.map(|query_id| format!("'{}'", query_id.replace('\'', "''")))
.collect::<Vec<_>>()
.join(", ");
format!(
"SELECT
COUNT(*),
COALESCE(SUM(COMPILATION_TIME), 0),
COALESCE(SUM(EXECUTION_TIME), 0)
FROM TABLE({}(
END_TIME_RANGE_START => DATEADD('second', -{lookback_seconds}, CURRENT_TIMESTAMP()),
RESULT_LIMIT => {QUERY_HISTORY_RESULT_LIMIT}
))
WHERE QUERY_ID IN ({identifiers})",
query_history_function(database),
)
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-core/src/cost_allocation/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,7 @@ mod allocation;
mod decimal;
#[path = "tests/helpers.rs"]
mod helpers;
#[path = "tests/test_statement_timings.rs"]
mod statement_timings;
#[path = "tests/test_types.rs"]
mod test_types;
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
use crate::cost_allocation::main::statement_timings_sql::statement_timings_sql;
use crate::cost_allocation::tests::test_types::StatementTimingsSqlTestCase;

#[test]
fn given_query_ids_when_rendering_timings_read_then_ids_are_quoted_and_database_qualified() {
let test_cases = [
StatementTimingsSqlTestCase {
description: "a quote in an id is doubled and the database is quoted upper case",
database: Some("analytics\"db"),
query_ids: vec!["01a-1", "it's"],
expected_sql: "SELECT
COUNT(*),
COALESCE(SUM(COMPILATION_TIME), 0),
COALESCE(SUM(EXECUTION_TIME), 0)
FROM TABLE(\"ANALYTICS\"\"DB\".INFORMATION_SCHEMA.QUERY_HISTORY(
END_TIME_RANGE_START => DATEADD('second', -720, CURRENT_TIMESTAMP()),
RESULT_LIMIT => 10000
))
WHERE QUERY_ID IN ('01a-1', 'it''s')",
},
StatementTimingsSqlTestCase {
description: "without a database the session's information schema is read",
database: None,
query_ids: vec!["q1"],
expected_sql: "SELECT
COUNT(*),
COALESCE(SUM(COMPILATION_TIME), 0),
COALESCE(SUM(EXECUTION_TIME), 0)
FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY(
END_TIME_RANGE_START => DATEADD('second', -720, CURRENT_TIMESTAMP()),
RESULT_LIMIT => 10000
))
WHERE QUERY_ID IN ('q1')",
},
];

for test_case in test_cases {
let query_ids: Vec<String> = test_case
.query_ids
.iter()
.map(|query_id| (*query_id).to_owned())
.collect();
assert_eq!(
statement_timings_sql(test_case.database, &query_ids, 720),
test_case.expected_sql,
"{}",
test_case.description
);
}
}
7 changes: 7 additions & 0 deletions crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,10 @@ pub(crate) struct AllocationTestCase {
pub(crate) expected_seconds: &'static str,
pub(crate) expected_usd: &'static str,
}

pub(crate) struct StatementTimingsSqlTestCase {
pub(crate) description: &'static str,
pub(crate) database: Option<&'static str>,
pub(crate) query_ids: Vec<&'static str>,
pub(crate) expected_sql: &'static str,
}
1 change: 1 addition & 0 deletions crates/sqlbuild-core/src/json/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
pub mod dumps;
pub mod python_float_repr;
pub mod with_section;
16 changes: 16 additions & 0 deletions crates/sqlbuild-core/src/json/main/with_section.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
//! Add a named section to a JSON document.

use serde_json::Value;

/// `payload`, a JSON document, with `section` added under `name`, pretty-printed with a trailing newline.
pub fn with_section(
payload: &str,
name: &str,
section: Value,
) -> Result<String, serde_json::Error> {
let mut document: Value = serde_json::from_str(payload)?;
if let Value::Object(fields) = &mut document {
fields.insert(name.to_owned(), section);
}
Ok(format!("{}\n", serde_json::to_string_pretty(&document)?))
}
15 changes: 15 additions & 0 deletions crates/sqlbuild-core/src/progress/_helpers/statements.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,18 @@ pub(crate) fn consume_statement_terminal(
projection.statement_query_ids.remove(statement_id);
(projection, line)
}

/// Count a finished statement by kind, keeping its query identifier.
pub(crate) fn count_statement(
mut projection: ProgressProjection,
event: &ProgressEvent,
) -> ProgressProjection {
let query_id: Option<String> = statement_query_id(&projection, event);
*projection
.statements
.by_kind
.entry(event.payload.statement_kind.clone())
.or_default() += 1;
projection.statements.query_ids.extend(query_id);
projection
}
3 changes: 3 additions & 0 deletions crates/sqlbuild-core/src/progress/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,3 +127,6 @@ pub(crate) const AUDIT_OUTCOME_STATUSES: [(&str, &str); 5] = [
("insufficient", "INSUFFICIENT"),
("skipped", "SKIP"),
];

/// Where reported statement compile and execution time comes from.
pub(crate) const STATEMENT_TIMING_SOURCE: &str = "snowflake_query_history";
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use crate::progress::_helpers::operations::{
use crate::progress::_helpers::resources::{consume_resource_start, consume_resource_terminal};
use crate::progress::_helpers::statements::{
consume_statement_start, consume_statement_submitted, consume_statement_terminal,
statement_heartbeat_line,
count_statement, statement_heartbeat_line,
};
use crate::progress::constants::{
OPERATION_START, RESOURCE_START, RETRY_SCHEDULED, STATEMENT_COMPLETED, STATEMENT_FAILED,
Expand Down Expand Up @@ -46,7 +46,7 @@ pub fn consume_progress_event(
let line: Option<String> = statement_heartbeat_line(&projection, event);
(projection, line)
} else if event_type == STATEMENT_COMPLETED || event_type == STATEMENT_FAILED {
consume_statement_terminal(projection, event)
consume_statement_terminal(count_statement(projection, event), event)
} else {
(projection, None)
};
Expand Down
45 changes: 45 additions & 0 deletions crates/sqlbuild-core/src/progress/main/format_statement_summary.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
//! The statement summary line of `sqb test` and `sqb scenario test`.

use crate::progress::models::{StatementSummary, StatementTiming};

/// `Statements: 12 (SELECT 8, CREATE 2, DROP 2); compile 1.5s, execution 0.8s`; nothing without statements.
#[must_use]
pub fn format_statement_summary(
summary: &StatementSummary,
timing: &StatementTiming,
) -> Option<String> {
let total: u64 = summary.by_kind.values().sum();
if total == 0 {
return None;
}
let mut kinds: Vec<(&String, &u64)> = summary.by_kind.iter().collect();
kinds.sort_by(|(left_kind, left), (right_kind, right)| {
right.cmp(left).then_with(|| left_kind.cmp(right_kind))
});
let kinds: String = kinds
.iter()
.map(|(kind, count)| format!("{kind} {count}"))
.collect::<Vec<_>>()
.join(", ");
let time: String = match timing {
StatementTiming::Reported(timings) => {
let identified: u64 = u64::try_from(summary.query_ids.len()).unwrap_or(u64::MAX);
let coverage: String = if timings.timed_statements < identified {
format!(
" ({} of {identified} statements in query history)",
timings.timed_statements
)
} else {
String::new()
};
format!(
"compile {:.1}s, execution {:.1}s{coverage}",
timings.compile_ms / 1000.0,
timings.execution_ms / 1000.0
)
}
StatementTiming::NotReported(adapter) => format!("compile time not reported by {adapter}"),
StatementTiming::Failed(reason) => format!("compile time unavailable ({reason})"),
};
Some(format!("Statements: {total} ({kinds}); {time}"))
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-core/src/progress/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ pub mod format_execution_header;
pub mod format_expectation_detail;
pub mod format_expectation_name;
pub mod format_inspection_record;
pub mod format_statement_summary;
pub mod format_status_spinner_line;
pub mod is_planning_completion_message;
pub mod is_planning_failure_message;
pub mod machine_output_mode;
pub mod progress_terminal_duration;
pub mod statement_summary_json;
30 changes: 30 additions & 0 deletions crates/sqlbuild-core/src/progress/main/statement_summary_json.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
//! The `statements` section of the `sqb test` and `sqb scenario test` JSON documents.

use serde_json::{Value, json};

use crate::progress::constants::STATEMENT_TIMING_SOURCE;
use crate::progress::models::{StatementSummary, StatementTiming};

/// `{total, by_kind, compile_time}`; `compile_time` is null when the adapter reports none.
#[must_use]
pub fn statement_summary_json(summary: &StatementSummary, timing: &StatementTiming) -> Value {
let compile_time: Value = match timing {
StatementTiming::Reported(timings) => json!({
"source": STATEMENT_TIMING_SOURCE,
"compile_ms": timings.compile_ms,
"execution_ms": timings.execution_ms,
"timed_statements": timings.timed_statements,
"identified_statements": summary.query_ids.len(),
}),
StatementTiming::NotReported(_) => Value::Null,
StatementTiming::Failed(reason) => json!({
"source": STATEMENT_TIMING_SOURCE,
"error": reason,
}),
};
json!({
"total": summary.by_kind.values().sum::<u64>(),
"by_kind": summary.by_kind,
"compile_time": compile_time,
})
}
35 changes: 34 additions & 1 deletion crates/sqlbuild-core/src/progress/models.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! The lifecycle facts progress projection reads, and the invocation projection state.

use std::collections::{HashMap, HashSet, VecDeque};
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};

use crate::progress::types::DifferenceSample;

Expand Down Expand Up @@ -83,6 +83,33 @@ pub struct ProgressProjection {
pub(crate) resource_names: HashMap<String, String>,
pub(crate) statement_starts: HashMap<String, ProgressEvent>,
pub(crate) statement_query_ids: HashMap<String, String>,
pub(crate) statements: StatementSummary,
}

/// The warehouse statements an invocation finished: how many of each leading keyword, and their query identifiers.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct StatementSummary {
pub by_kind: BTreeMap<String, u64>,
pub query_ids: Vec<String>,
}

/// Warehouse-reported time of the statements found in query history.
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct StatementTimings {
pub compile_ms: f64,
pub execution_ms: f64,
/// How many of the summary's query identifiers query history returned.
pub timed_statements: u64,
}

/// What the warehouse reports about the time of an invocation's statements.
#[derive(Clone, Debug, PartialEq)]
pub enum StatementTiming {
Reported(StatementTimings),
/// The adapter, by name, keeps no per-statement compile time.
NotReported(String),
/// Reading query history failed, for this reason.
Failed(String),
}

impl ProgressProjection {
Expand Down Expand Up @@ -116,6 +143,12 @@ impl ProgressProjection {
.insert(resource_attempt_id.to_owned(), outcome.to_owned());
}

/// The warehouse statements finished so far.
#[must_use]
pub fn statement_summary(&self) -> &StatementSummary {
&self.statements
}

/// Whether a visible operation with this name currently owns progress.
#[must_use]
pub fn is_operation_active(&self, operation_name: &str) -> bool {
Expand Down
23 changes: 23 additions & 0 deletions crates/sqlbuild-core/src/progress/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,3 +115,26 @@ pub(super) fn audit_rows(test_case: &AuditRowTestCase) -> Vec<String> {
}
lines
}

/// A statement event of `kind`, inside the `orders` attempt `attempt_id` or outside any resource.
pub(super) fn statement_event(
(event_id, event_type): (&str, &str),
(statement_id, kind, query_id): (&str, &str, Option<&str>),
attempt_id: Option<&str>,
) -> ProgressEvent {
ProgressEvent {
event_id: event_id.to_owned(),
event_type: event_type.to_owned(),
resource_id: attempt_id.map(|_| "model:orders".to_owned()),
resource_attempt_id: attempt_id.map(str::to_owned),
statement_id: Some(statement_id.to_owned()),
payload: ProgressPayload {
resource_name: attempt_id.map(|_| "orders".to_owned()),
resource_kind: attempt_id.map(|_| "table".to_owned()),
statement_kind: kind.to_owned(),
query_id: query_id.map(str::to_owned),
..ProgressPayload::default()
},
..ProgressEvent::default()
}
}
Loading
Loading