diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/history_function.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/history_function.rs new file mode 100644 index 0000000000..9cb38a0a0d --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/history_function.rs @@ -0,0 +1,14 @@ +//! The Snowflake `QUERY_HISTORY` table function, qualified by database when there is one. + +use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION; + +/// `INFORMATION_SCHEMA.QUERY_HISTORY`, under `database` when it is given. +pub(crate) fn query_history_function(database: Option<&str>) -> String { + match database { + None => QUERY_HISTORY_FUNCTION.to_owned(), + Some(database) => format!( + "\"{}\".{QUERY_HISTORY_FUNCTION}", + database.to_uppercase().replace('"', "\"\"") + ), + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs index a9f8414daa..309c918ce6 100644 --- a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod allocation; pub(crate) mod decimal; +pub(crate) mod history_function; pub(crate) mod query_text; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/mod.rs b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs index d4b3364262..b0a502a247 100644 --- a/crates/sqlbuild-core/src/cost_allocation/main/mod.rs +++ b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs @@ -1,2 +1,3 @@ pub mod allocate_run_cost; pub mod query_history_sql; +pub mod statement_timings_sql; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs index 54fa520146..16b0b1ae7f 100644 --- a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs +++ b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs @@ -1,19 +1,9 @@ //! `query_history_sql`: the Snowflake `QUERY_HISTORY` read of one end-time window. +use crate::cost_allocation::_helpers::history_function::query_history_function; use crate::cost_allocation::_helpers::query_text::query_text_expression; -use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION; use crate::cost_allocation::models::QueryHistoryWindow; -fn query_history_function(database: Option<&str>) -> String { - match database { - None => QUERY_HISTORY_FUNCTION.to_owned(), - Some(database) => format!( - "\"{}\".{QUERY_HISTORY_FUNCTION}", - database.to_uppercase().replace('"', "\"\"") - ), - } -} - /// Query-history SQL for queries ending after `start`, and before `end` when it is bounded. pub fn query_history_sql(window: &QueryHistoryWindow<'_>) -> String { let end_range: String = match window.end { diff --git a/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs new file mode 100644 index 0000000000..6ccc8ac2a0 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs @@ -0,0 +1,30 @@ +//! `statement_timings_sql`: compile and execution time of known Snowflake queries. + +use crate::cost_allocation::_helpers::history_function::query_history_function; +use crate::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; + +/// One query-history read summing compile and execution milliseconds of `query_ids` that ended in the last `lookback_seconds`. +#[must_use] +pub fn statement_timings_sql( + database: Option<&str>, + query_ids: &[String], + lookback_seconds: u64, +) -> String { + let identifiers: String = query_ids + .iter() + .map(|query_id| format!("'{}'", query_id.replace('\'', "''"))) + .collect::>() + .join(", "); + format!( + "SELECT + COUNT(*), + COALESCE(SUM(COMPILATION_TIME), 0), + COALESCE(SUM(EXECUTION_TIME), 0) +FROM TABLE({}( + END_TIME_RANGE_START => DATEADD('second', -{lookback_seconds}, CURRENT_TIMESTAMP()), + RESULT_LIMIT => {QUERY_HISTORY_RESULT_LIMIT} +)) +WHERE QUERY_ID IN ({identifiers})", + query_history_function(database), + ) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests.rs b/crates/sqlbuild-core/src/cost_allocation/tests.rs index 6949838ed8..a1d089ca93 100644 --- a/crates/sqlbuild-core/src/cost_allocation/tests.rs +++ b/crates/sqlbuild-core/src/cost_allocation/tests.rs @@ -4,5 +4,7 @@ mod allocation; mod decimal; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_statement_timings.rs"] +mod statement_timings; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs new file mode 100644 index 0000000000..f6993b2d8b --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs @@ -0,0 +1,50 @@ +use crate::cost_allocation::main::statement_timings_sql::statement_timings_sql; +use crate::cost_allocation::tests::test_types::StatementTimingsSqlTestCase; + +#[test] +fn given_query_ids_when_rendering_timings_read_then_ids_are_quoted_and_database_qualified() { + let test_cases = [ + StatementTimingsSqlTestCase { + description: "a quote in an id is doubled and the database is quoted upper case", + database: Some("analytics\"db"), + query_ids: vec!["01a-1", "it's"], + expected_sql: "SELECT + COUNT(*), + COALESCE(SUM(COMPILATION_TIME), 0), + COALESCE(SUM(EXECUTION_TIME), 0) +FROM TABLE(\"ANALYTICS\"\"DB\".INFORMATION_SCHEMA.QUERY_HISTORY( + END_TIME_RANGE_START => DATEADD('second', -720, CURRENT_TIMESTAMP()), + RESULT_LIMIT => 10000 +)) +WHERE QUERY_ID IN ('01a-1', 'it''s')", + }, + StatementTimingsSqlTestCase { + description: "without a database the session's information schema is read", + database: None, + query_ids: vec!["q1"], + expected_sql: "SELECT + COUNT(*), + COALESCE(SUM(COMPILATION_TIME), 0), + COALESCE(SUM(EXECUTION_TIME), 0) +FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY( + END_TIME_RANGE_START => DATEADD('second', -720, CURRENT_TIMESTAMP()), + RESULT_LIMIT => 10000 +)) +WHERE QUERY_ID IN ('q1')", + }, + ]; + + for test_case in test_cases { + let query_ids: Vec = test_case + .query_ids + .iter() + .map(|query_id| (*query_id).to_owned()) + .collect(); + assert_eq!( + statement_timings_sql(test_case.database, &query_ids, 720), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs index cefbc507fe..9c6ffe2fee 100644 --- a/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs @@ -12,3 +12,10 @@ pub(crate) struct AllocationTestCase { pub(crate) expected_seconds: &'static str, pub(crate) expected_usd: &'static str, } + +pub(crate) struct StatementTimingsSqlTestCase { + pub(crate) description: &'static str, + pub(crate) database: Option<&'static str>, + pub(crate) query_ids: Vec<&'static str>, + pub(crate) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-core/src/json/main/mod.rs b/crates/sqlbuild-core/src/json/main/mod.rs index c295400d51..18b8ecee33 100644 --- a/crates/sqlbuild-core/src/json/main/mod.rs +++ b/crates/sqlbuild-core/src/json/main/mod.rs @@ -1,2 +1,3 @@ pub mod dumps; pub mod python_float_repr; +pub mod with_section; diff --git a/crates/sqlbuild-core/src/json/main/with_section.rs b/crates/sqlbuild-core/src/json/main/with_section.rs new file mode 100644 index 0000000000..9a818f4ef8 --- /dev/null +++ b/crates/sqlbuild-core/src/json/main/with_section.rs @@ -0,0 +1,16 @@ +//! Add a named section to a JSON document. + +use serde_json::Value; + +/// `payload`, a JSON document, with `section` added under `name`, pretty-printed with a trailing newline. +pub fn with_section( + payload: &str, + name: &str, + section: Value, +) -> Result { + let mut document: Value = serde_json::from_str(payload)?; + if let Value::Object(fields) = &mut document { + fields.insert(name.to_owned(), section); + } + Ok(format!("{}\n", serde_json::to_string_pretty(&document)?)) +} diff --git a/crates/sqlbuild-core/src/progress/_helpers/statements.rs b/crates/sqlbuild-core/src/progress/_helpers/statements.rs index 76e95d91c5..296874db69 100644 --- a/crates/sqlbuild-core/src/progress/_helpers/statements.rs +++ b/crates/sqlbuild-core/src/progress/_helpers/statements.rs @@ -142,3 +142,18 @@ pub(crate) fn consume_statement_terminal( projection.statement_query_ids.remove(statement_id); (projection, line) } + +/// Count a finished statement by kind, keeping its query identifier. +pub(crate) fn count_statement( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> ProgressProjection { + let query_id: Option = statement_query_id(&projection, event); + *projection + .statements + .by_kind + .entry(event.payload.statement_kind.clone()) + .or_default() += 1; + projection.statements.query_ids.extend(query_id); + projection +} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index bcdf9d0799..e8289a5288 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -127,3 +127,6 @@ pub(crate) const AUDIT_OUTCOME_STATUSES: [(&str, &str); 5] = [ ("insufficient", "INSUFFICIENT"), ("skipped", "SKIP"), ]; + +/// Where reported statement compile and execution time comes from. +pub(crate) const STATEMENT_TIMING_SOURCE: &str = "snowflake_query_history"; diff --git a/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs index 09c10fccd3..744b1fa2bb 100644 --- a/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs +++ b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs @@ -6,7 +6,7 @@ use crate::progress::_helpers::operations::{ use crate::progress::_helpers::resources::{consume_resource_start, consume_resource_terminal}; use crate::progress::_helpers::statements::{ consume_statement_start, consume_statement_submitted, consume_statement_terminal, - statement_heartbeat_line, + count_statement, statement_heartbeat_line, }; use crate::progress::constants::{ OPERATION_START, RESOURCE_START, RETRY_SCHEDULED, STATEMENT_COMPLETED, STATEMENT_FAILED, @@ -46,7 +46,7 @@ pub fn consume_progress_event( let line: Option = statement_heartbeat_line(&projection, event); (projection, line) } else if event_type == STATEMENT_COMPLETED || event_type == STATEMENT_FAILED { - consume_statement_terminal(projection, event) + consume_statement_terminal(count_statement(projection, event), event) } else { (projection, None) }; diff --git a/crates/sqlbuild-core/src/progress/main/format_statement_summary.rs b/crates/sqlbuild-core/src/progress/main/format_statement_summary.rs new file mode 100644 index 0000000000..c189b1de47 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/format_statement_summary.rs @@ -0,0 +1,45 @@ +//! The statement summary line of `sqb test` and `sqb scenario test`. + +use crate::progress::models::{StatementSummary, StatementTiming}; + +/// `Statements: 12 (SELECT 8, CREATE 2, DROP 2); compile 1.5s, execution 0.8s`; nothing without statements. +#[must_use] +pub fn format_statement_summary( + summary: &StatementSummary, + timing: &StatementTiming, +) -> Option { + let total: u64 = summary.by_kind.values().sum(); + if total == 0 { + return None; + } + let mut kinds: Vec<(&String, &u64)> = summary.by_kind.iter().collect(); + kinds.sort_by(|(left_kind, left), (right_kind, right)| { + right.cmp(left).then_with(|| left_kind.cmp(right_kind)) + }); + let kinds: String = kinds + .iter() + .map(|(kind, count)| format!("{kind} {count}")) + .collect::>() + .join(", "); + let time: String = match timing { + StatementTiming::Reported(timings) => { + let identified: u64 = u64::try_from(summary.query_ids.len()).unwrap_or(u64::MAX); + let coverage: String = if timings.timed_statements < identified { + format!( + " ({} of {identified} statements in query history)", + timings.timed_statements + ) + } else { + String::new() + }; + format!( + "compile {:.1}s, execution {:.1}s{coverage}", + timings.compile_ms / 1000.0, + timings.execution_ms / 1000.0 + ) + } + StatementTiming::NotReported(adapter) => format!("compile time not reported by {adapter}"), + StatementTiming::Failed(reason) => format!("compile time unavailable ({reason})"), + }; + Some(format!("Statements: {total} ({kinds}); {time}")) +} diff --git a/crates/sqlbuild-core/src/progress/main/mod.rs b/crates/sqlbuild-core/src/progress/main/mod.rs index e8f7c3c6f5..c912a2b982 100644 --- a/crates/sqlbuild-core/src/progress/main/mod.rs +++ b/crates/sqlbuild-core/src/progress/main/mod.rs @@ -8,8 +8,10 @@ pub mod format_execution_header; pub mod format_expectation_detail; pub mod format_expectation_name; pub mod format_inspection_record; +pub mod format_statement_summary; pub mod format_status_spinner_line; pub mod is_planning_completion_message; pub mod is_planning_failure_message; pub mod machine_output_mode; pub mod progress_terminal_duration; +pub mod statement_summary_json; diff --git a/crates/sqlbuild-core/src/progress/main/statement_summary_json.rs b/crates/sqlbuild-core/src/progress/main/statement_summary_json.rs new file mode 100644 index 0000000000..a5f7f02cc1 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/statement_summary_json.rs @@ -0,0 +1,30 @@ +//! The `statements` section of the `sqb test` and `sqb scenario test` JSON documents. + +use serde_json::{Value, json}; + +use crate::progress::constants::STATEMENT_TIMING_SOURCE; +use crate::progress::models::{StatementSummary, StatementTiming}; + +/// `{total, by_kind, compile_time}`; `compile_time` is null when the adapter reports none. +#[must_use] +pub fn statement_summary_json(summary: &StatementSummary, timing: &StatementTiming) -> Value { + let compile_time: Value = match timing { + StatementTiming::Reported(timings) => json!({ + "source": STATEMENT_TIMING_SOURCE, + "compile_ms": timings.compile_ms, + "execution_ms": timings.execution_ms, + "timed_statements": timings.timed_statements, + "identified_statements": summary.query_ids.len(), + }), + StatementTiming::NotReported(_) => Value::Null, + StatementTiming::Failed(reason) => json!({ + "source": STATEMENT_TIMING_SOURCE, + "error": reason, + }), + }; + json!({ + "total": summary.by_kind.values().sum::(), + "by_kind": summary.by_kind, + "compile_time": compile_time, + }) +} diff --git a/crates/sqlbuild-core/src/progress/models.rs b/crates/sqlbuild-core/src/progress/models.rs index a2808e6b64..f9d0f1612a 100644 --- a/crates/sqlbuild-core/src/progress/models.rs +++ b/crates/sqlbuild-core/src/progress/models.rs @@ -1,6 +1,6 @@ //! The lifecycle facts progress projection reads, and the invocation projection state. -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use crate::progress::types::DifferenceSample; @@ -83,6 +83,33 @@ pub struct ProgressProjection { pub(crate) resource_names: HashMap, pub(crate) statement_starts: HashMap, pub(crate) statement_query_ids: HashMap, + pub(crate) statements: StatementSummary, +} + +/// The warehouse statements an invocation finished: how many of each leading keyword, and their query identifiers. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StatementSummary { + pub by_kind: BTreeMap, + pub query_ids: Vec, +} + +/// Warehouse-reported time of the statements found in query history. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct StatementTimings { + pub compile_ms: f64, + pub execution_ms: f64, + /// How many of the summary's query identifiers query history returned. + pub timed_statements: u64, +} + +/// What the warehouse reports about the time of an invocation's statements. +#[derive(Clone, Debug, PartialEq)] +pub enum StatementTiming { + Reported(StatementTimings), + /// The adapter, by name, keeps no per-statement compile time. + NotReported(String), + /// Reading query history failed, for this reason. + Failed(String), } impl ProgressProjection { @@ -116,6 +143,12 @@ impl ProgressProjection { .insert(resource_attempt_id.to_owned(), outcome.to_owned()); } + /// The warehouse statements finished so far. + #[must_use] + pub fn statement_summary(&self) -> &StatementSummary { + &self.statements + } + /// Whether a visible operation with this name currently owns progress. #[must_use] pub fn is_operation_active(&self, operation_name: &str) -> bool { diff --git a/crates/sqlbuild-core/src/progress/tests/helpers.rs b/crates/sqlbuild-core/src/progress/tests/helpers.rs index bdbd5d6d28..a3367b93a7 100644 --- a/crates/sqlbuild-core/src/progress/tests/helpers.rs +++ b/crates/sqlbuild-core/src/progress/tests/helpers.rs @@ -115,3 +115,26 @@ pub(super) fn audit_rows(test_case: &AuditRowTestCase) -> Vec { } lines } + +/// A statement event of `kind`, inside the `orders` attempt `attempt_id` or outside any resource. +pub(super) fn statement_event( + (event_id, event_type): (&str, &str), + (statement_id, kind, query_id): (&str, &str, Option<&str>), + attempt_id: Option<&str>, +) -> ProgressEvent { + ProgressEvent { + event_id: event_id.to_owned(), + event_type: event_type.to_owned(), + resource_id: attempt_id.map(|_| "model:orders".to_owned()), + resource_attempt_id: attempt_id.map(str::to_owned), + statement_id: Some(statement_id.to_owned()), + payload: ProgressPayload { + resource_name: attempt_id.map(|_| "orders".to_owned()), + resource_kind: attempt_id.map(|_| "table".to_owned()), + statement_kind: kind.to_owned(), + query_id: query_id.map(str::to_owned), + ..ProgressPayload::default() + }, + ..ProgressEvent::default() + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress.rs b/crates/sqlbuild-core/src/progress/tests/test_progress.rs index 9ffb9e3b11..0ea684b0e0 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress.rs @@ -1,8 +1,18 @@ -use crate::progress::models::{ProgressEvent, ProgressPayload, ProgressSettings}; +use std::collections::BTreeMap; + +use crate::progress::main::consume_progress_event::consume_progress_event; +use crate::progress::main::format_statement_summary::format_statement_summary; +use crate::progress::main::statement_summary_json::statement_summary_json; +use crate::progress::models::{ + ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, StatementSummary, + StatementTiming, StatementTimings, +}; use crate::progress::tests::helpers::{ - attempt_event, audit_rows, operation_event, orders_claim, plain, project_case, + attempt_event, audit_rows, operation_event, orders_claim, plain, project_case, statement_event, +}; +use crate::progress::tests::test_types::{ + AuditRowTestCase, ProjectionTestCase, StatementCountTestCase, StatementSummaryTextTestCase, }; -use crate::progress::tests::test_types::{AuditRowTestCase, ProjectionTestCase}; #[test] fn given_lifecycle_facts_when_projecting_then_progress_lines_match() { @@ -265,3 +275,231 @@ fn given_standalone_audit_when_projecting_then_row_names_target_and_outcome() { ); } } + +#[test] +fn given_statement_terminals_when_projecting_then_every_finished_statement_is_counted_once() { + let test_cases = [ + StatementCountTestCase { + description: "a known resource's terminal takes the query id its submission reported", + events: vec![ + attempt_event("r1", "resource_attempt_started", ("a1", None)), + statement_event( + ("s1", "statement_started"), + ("st1", "CREATE", None), + Some("a1"), + ), + statement_event( + ("s2", "statement_submitted"), + ("st1", "CREATE", Some("q1")), + Some("a1"), + ), + statement_event( + ("s3", "statement_completed"), + ("st1", "CREATE", None), + Some("a1"), + ), + ], + expected_by_kind: vec![("CREATE", 1)], + expected_query_ids: vec!["q1"], + }, + StatementCountTestCase { + description: "a redelivered terminal is counted once", + events: vec![ + statement_event( + ("s1", "statement_completed"), + ("st1", "WITH", Some("q1")), + None, + ), + statement_event( + ("s1", "statement_completed"), + ("st1", "WITH", Some("q1")), + None, + ), + ], + expected_by_kind: vec![("WITH", 1)], + expected_query_ids: vec!["q1"], + }, + StatementCountTestCase { + description: "statements outside known resources count, failed ones too", + events: vec![ + statement_event( + ("s1", "statement_completed"), + ("st1", "WITH", Some("q1")), + None, + ), + statement_event(("s2", "statement_failed"), ("st2", "DROP", None), None), + statement_event( + ("s3", "statement_completed"), + ("st3", "WITH", Some("q3")), + None, + ), + ], + expected_by_kind: vec![("DROP", 1), ("WITH", 2)], + expected_query_ids: vec!["q1", "q3"], + }, + StatementCountTestCase { + description: "starts, submissions and heartbeats alone are not finished statements", + events: vec![ + statement_event(("s1", "statement_started"), ("st1", "SELECT", None), None), + statement_event( + ("s2", "statement_submitted"), + ("st1", "SELECT", Some("q1")), + None, + ), + statement_event( + ("s3", "statement_heartbeat"), + ("st1", "SELECT", Some("q1")), + None, + ), + ], + expected_by_kind: vec![], + expected_query_ids: vec![], + }, + ]; + + for test_case in test_cases { + let mut projection: ProgressProjection = ProgressProjection::new(plain()); + for event in &test_case.events { + projection = consume_progress_event(projection, event).0; + } + let summary: &StatementSummary = projection.statement_summary(); + assert_eq!( + (summary.by_kind.clone(), summary.query_ids.clone()), + ( + test_case + .expected_by_kind + .iter() + .map(|(kind, count)| ((*kind).to_owned(), *count)) + .collect::>(), + test_case + .expected_query_ids + .iter() + .map(|query_id| (*query_id).to_owned()) + .collect::>(), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_statement_summary_when_formatting_then_line_and_json_report_kinds_and_timing() { + let summary: StatementSummary = StatementSummary { + by_kind: BTreeMap::from([ + ("CREATE".to_owned(), 2), + ("DROP".to_owned(), 2), + ("SELECT".to_owned(), 7), + ("ALTER".to_owned(), 1), + ]), + query_ids: (0..12).map(|index| format!("q{index}")).collect(), + }; + let reported = |compile_ms: f64, execution_ms: f64, timed_statements: u64| { + StatementTiming::Reported(StatementTimings { + compile_ms, + execution_ms, + timed_statements, + }) + }; + let test_cases = [ + StatementSummaryTextTestCase { + description: "kinds run from most to least frequent, ties by name", + summary: summary.clone(), + timing: reported(1540.0, 260.0, 12), + expected_line: Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s", + ), + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": { + "source": "snowflake_query_history", + "compile_ms": 1540.0, + "execution_ms": 260.0, + "timed_statements": 12, + "identified_statements": 12, + }, + }), + }, + StatementSummaryTextTestCase { + description: "statements missing from query history are called out", + summary: summary.clone(), + timing: reported(1000.0, 0.0, 10), + expected_line: Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.0s, execution 0.0s (10 of 12 statements in query history)", + ), + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": { + "source": "snowflake_query_history", + "compile_ms": 1000.0, + "execution_ms": 0.0, + "timed_statements": 10, + "identified_statements": 12, + }, + }), + }, + StatementSummaryTextTestCase { + description: "an adapter without query history says so", + summary: summary.clone(), + timing: StatementTiming::NotReported("duckdb".to_owned()), + expected_line: Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile time not reported by duckdb", + ), + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": null, + }), + }, + StatementSummaryTextTestCase { + description: "a failed read says why", + summary: summary.clone(), + timing: StatementTiming::Failed("query history read failed".to_owned()), + expected_line: Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile time unavailable (query history read failed)", + ), + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": { + "source": "snowflake_query_history", + "error": "query history read failed", + }, + }), + }, + StatementSummaryTextTestCase { + description: "a run without statements has no line", + summary: StatementSummary::default(), + timing: reported(0.0, 0.0, 0), + expected_line: None, + expected_json: serde_json::json!({ + "total": 0, + "by_kind": {}, + "compile_time": { + "source": "snowflake_query_history", + "compile_ms": 0.0, + "execution_ms": 0.0, + "timed_statements": 0, + "identified_statements": 0, + }, + }), + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + format_statement_summary(&test_case.summary, &test_case.timing), + statement_summary_json(&test_case.summary, &test_case.timing), + ), + ( + test_case.expected_line.map(str::to_owned), + test_case.expected_json, + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs index 1606615f18..ac0b995f26 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_types.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::cli_text::models::CliStyle; -use crate::progress::models::{ProgressEvent, ProgressSettings}; +use crate::progress::models::{ProgressEvent, ProgressSettings, StatementSummary, StatementTiming}; use crate::terminal_events::models::TerminalRequest; pub(super) struct ProjectionTestCase { @@ -57,3 +57,18 @@ pub(super) struct AuditRowTestCase { pub(super) recorded_outcomes: Vec<(&'static str, &'static str)>, pub(super) expected_lines: Vec<&'static str>, } + +pub(super) struct StatementCountTestCase { + pub(super) description: &'static str, + pub(super) events: Vec, + pub(super) expected_by_kind: Vec<(&'static str, u64)>, + pub(super) expected_query_ids: Vec<&'static str>, +} + +pub(super) struct StatementSummaryTextTestCase { + pub(super) description: &'static str, + pub(super) summary: StatementSummary, + pub(super) timing: StatementTiming, + pub(super) expected_line: Option<&'static str>, + pub(super) expected_json: serde_json::Value, +} diff --git a/crates/sqlbuild-python-base/src/build_progress/models.rs b/crates/sqlbuild-python-base/src/build_progress/models.rs index dd2c38c3b9..36b5e2b04c 100644 --- a/crates/sqlbuild-python-base/src/build_progress/models.rs +++ b/crates/sqlbuild-python-base/src/build_progress/models.rs @@ -47,6 +47,7 @@ use sqlbuild_core::progress::main::progress_terminal_duration::progress_terminal use sqlbuild_core::progress::models::ProgressEvent; use sqlbuild_core::progress::models::ProgressProjection; use sqlbuild_core::progress::models::ProgressSettings; +use sqlbuild_core::progress::models::StatementSummary; use sqlbuild_core::terminal_events::models::TerminalRequest; use std::sync::Mutex; use std::sync::MutexGuard; @@ -130,6 +131,11 @@ impl NativeProgressProjector { lock.call_method0("release")?; result } + + /// The warehouse statements this invocation finished so far. + pub fn statement_summary(&self) -> PyResult { + Ok(self.locked()?.statement_summary().clone()) + } } #[pymethods] diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index d77c2bcff3..3ac29f7c87 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -150,6 +150,23 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), ], }; +const R_TARGET_SETTINGS: RecordSpec = RecordSpec { + name: "TargetSettings", + doc: "The active target's database and schemas, resolved as SQLBuild uses them.", + fields: &[ + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("loader_schema", FieldDefault::None), + FieldSpec::new("database_preserved", FieldDefault::False), + FieldSpec::new("schema_preserved", FieldDefault::False), + ], +}; +const R_PROVIDER_SETUP_CONTEXT: RecordSpec = RecordSpec { + name: "ProviderSetupContext", + doc: "What a provider's setup receives: the active target's resolved settings.", + fields: &[FieldSpec::new("target_config", FieldDefault::None)], +}; const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { name: "LocalTargetConfig", doc: "Local developer overrides for one named target.", @@ -729,6 +746,8 @@ pub(crate) const CONTRACT_RECORDS: &[RecordSpec] = &[ R_TARGET_DIFF_CONFIG, R_TARGET_WAREHOUSES_CONFIG, R_TARGET_CONFIG, + R_TARGET_SETTINGS, + R_PROVIDER_SETUP_CONTEXT, R_LOCAL_TARGET_CONFIG, R_SETTINGS_CONFIG, R_SCOPES_CONFIG, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs index 36b3d857ae..3a3a2c281d 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs @@ -398,6 +398,7 @@ native_id_names! { RecordId::Project => "Project", RecordId::ProjectConfig => "ProjectConfig", RecordId::ProjectPath => "ProjectPath", + RecordId::ProviderSetupContext => "ProviderSetupContext", RecordId::PythonHookEntry => "PythonHookEntry", RecordId::PythonIdentityDependency => "PythonIdentityDependency", RecordId::PythonIdentityStatus => "PythonIdentityStatus", @@ -567,6 +568,7 @@ native_id_names! { RecordId::TableTypeSource => "TableTypeSource", RecordId::TargetConfig => "TargetConfig", RecordId::TargetDiffConfig => "TargetDiffConfig", + RecordId::TargetSettings => "TargetSettings", RecordId::TargetWarehousesConfig => "TargetWarehousesConfig", RecordId::TestCommandRequest => "TestCommandRequest", RecordId::TestInvocation => "TestInvocation", @@ -689,7 +691,6 @@ native_id_names! { NativeAttr::BuildEffectiveRuntimeConfig => "build_effective_runtime_config", NativeAttr::BuildEffectiveSettingsConfig => "build_effective_settings_config", NativeAttr::BuildEffectiveTargetConfig => "build_effective_target_config", - NativeAttr::BuildEffectiveTargetNamespace => "build_effective_target_namespace", NativeAttr::BuildExecutionPlan => "build_execution_plan", NativeAttr::BuildFailedSqlbuildModelNames => "build_failed_sqlbuild_model_names", NativeAttr::BuildIdentity => "BUILD_IDENTITY", diff --git a/crates/sqlbuild-python-base/src/records/types.rs b/crates/sqlbuild-python-base/src/records/types.rs index 7d9043d0cb..8bbe5811e5 100644 --- a/crates/sqlbuild-python-base/src/records/types.rs +++ b/crates/sqlbuild-python-base/src/records/types.rs @@ -391,6 +391,7 @@ pub enum RecordId { Project, ProjectConfig, ProjectPath, + ProviderSetupContext, PythonHookEntry, PythonIdentityDependency, PythonIdentityStatus, @@ -560,6 +561,7 @@ pub enum RecordId { TableTypeSource, TargetConfig, TargetDiffConfig, + TargetSettings, TargetWarehousesConfig, TestCommandRequest, TestInvocation, @@ -682,7 +684,6 @@ pub enum NativeAttr { BuildEffectiveRuntimeConfig, BuildEffectiveSettingsConfig, BuildEffectiveTargetConfig, - BuildEffectiveTargetNamespace, BuildExecutionPlan, BuildFailedSqlbuildModelNames, BuildIdentity, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 5faa65a20e..08bb455cde 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,6 +19,9 @@ use crate::bindings::_helpers::build_command::planning::{ write_compile_target, write_plan_text, }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, selected_target_settings, +}; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, }; @@ -30,6 +33,7 @@ use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::diagnostics::models::BuildPhaseTimingTracker; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; @@ -39,12 +43,31 @@ use sqlbuild_python_project::project_snapshot::models::WorkerBuild; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { let py: Python<'_> = context.py(); + let discovered: Bound<'_, PyAny> = context.get("discovered_inputs")?; + let selected_target: Bound<'_, PyAny> = native_field(&context.request, "selected_target")?; + let (_, effective_vars, _): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = + build_effective_runtime_config( + &discovered, + Some(selected_target.clone()).filter(|value| !value.is_none()), + Some(native_field(&context.request, "cli_vars")?).filter(|value| !value.is_none()), + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? + .extract()?; + let target_config: TargetSettingsValues = selected_target_settings( + (&discovered, &selected_target), + &context.get("adapter_name")?.extract::()?, + ( + &context.get("connection_config")?, + &effective_vars, + &context.get("adapter")?, + ), + )?; let session: Bound<'_, PyAny> = call_with( &host_entry(py, UserCodeHost::BuildProviderSession)?, - &[( - "discovered_providers", - context.get("discovered_inputs")?.getattr("providers")?, - )], + &[ + ("discovered_providers", discovered.getattr("providers")?), + ("setup_context", target_config.setup_context(py)?), + ], )?; let built: PyResult = run_planned_build(context, &native_field(&session, "providers")?); let closed: PyResult> = session.call_method0("close"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 22c526cf64..c633991b0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::check::check_core::{ use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; +use crate::bindings::_helpers::compile_driver::targets::project_target_settings; use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; use crate::bindings::_helpers::consumption::recorder::{ ConsumptionRecorder, Prepared, held_back_text, prepare as prepare_consumption, run_reason_text, @@ -357,6 +358,15 @@ pub(crate) fn execute_build_plan<'py>( preparation.concurrency.into_pyobject(py)?.into_any(), ), ("loader_is_reload", request("reload_sources")?), + ( + "target_config", + project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&connection_config, &adapter), + )? + .record(py)?, + ), ("start_cursor_ts", start_ts.clone()), ("end_cursor_ts", end_ts.clone()), ("start_cursor_int", start_int.clone()), @@ -634,6 +644,15 @@ fn check_runtime<'py>( "is_reload", native_field(&context.request, "reload_sources")?, ), + ( + "target_config", + project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&context.get("connection_config")?, &adapter), + )? + .record(py)?, + ), ( "default_database", adapter.call_method0("default_database")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index c073f3eaac..7e7b13b14b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -14,6 +14,9 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, project_target_settings, +}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -259,6 +262,7 @@ struct RuntimeParts<'a, 'py> { project: &'a Bound<'py, PyAny>, reload: bool, defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), + target_config: TargetSettingsValues, relations: (Bound<'py, PyAny>, Bound<'py, PyAny>), inputs: &'a LifecycleInputs<'py>, } @@ -282,6 +286,7 @@ fn node_runtime<'py>( ), ("vars", native_field(parts.project, "effective_vars")?), ("is_reload", boolean(py, parts.reload)), + ("target_config", parts.target_config.record(py)?), ("default_database", parts.defaults.0.clone()), ("default_schema", parts.defaults.1.clone()), ("start_cursor_ts", start_ts.clone()), @@ -441,6 +446,11 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( adapter.call_method0("default_schema") })?, ), + target_config: project_target_settings( + &discovered, + &project, + (&connection_config, &adapter), + )?, relations: (relation_targets, project_relations), inputs, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 293c886880..9d570e4062 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -6,7 +6,6 @@ use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; -use serde_json::Value; use sqlbuild_analysis::definition_changes::constants::{ MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, TEST_FILE_KIND, @@ -20,6 +19,7 @@ use sqlbuild_analysis::definition_changes::main::selection_report::selection_rep use sqlbuild_analysis::definition_changes::models::{ DefinitionChanges, DefinitionSnapshot, ExecutedItem, SelectedItem, }; +use sqlbuild_core::json::main::with_section::with_section; use crate::bindings::_helpers::change_selection::base_snapshot::BaseComparison; use crate::bindings::_helpers::change_selection::definition_snapshot::{ @@ -311,17 +311,12 @@ pub(crate) fn with_since_section(payload: String, since: Option<&str>) -> PyResu return Ok(payload); }; let invalid = |error: serde_json::Error| PyValueError::new_err(error.to_string()); - let mut document: Value = serde_json::from_str(&payload).map_err(invalid)?; - if let Value::Object(fields) = &mut document { - fields.insert( - SINCE_SECTION.to_owned(), - serde_json::from_str(since).map_err(invalid)?, - ); - } - Ok(format!( - "{}\n", - serde_json::to_string_pretty(&document).map_err(invalid)? - )) + with_section( + &payload, + SINCE_SECTION, + serde_json::from_str(since).map_err(invalid)?, + ) + .map_err(invalid) } /// The project's compiled tests and models by name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 14347b476b..cb3d90f5f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -7,6 +7,9 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pi use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, project_target_settings, +}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; @@ -70,6 +73,7 @@ struct Prepared<'py> { relation_targets: Bound<'py, PyAny>, relation_refs: Bound<'py, PyAny>, defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), + target_config: TargetSettingsValues, project_relations: Bound<'py, PyAny>, } @@ -354,7 +358,13 @@ fn prepare<'py>( )?), )?, )?; + let target_config: TargetSettingsValues = project_target_settings( + &discovered, + &project, + (&native_field(invocation, "connection_config")?, &adapter), + )?; Ok(Prepared { + target_config, check_functions: PyTuple::new(py, check_functions)?.into_any(), lifecycle_plan, relation_targets, @@ -405,6 +415,7 @@ fn node_runtime<'py>( ("target", native_field(&project, "effective_target_name")?), ("vars", native_field(&project, "effective_vars")?), ("is_reload", PyBool::new(py, false).to_owned().into_any()), + ("target_config", prepared.target_config.record(py)?), ("default_database", prepared.defaults.0.clone()), ("default_schema", prepared.defaults.1.clone()), ("relation_targets", prepared.relation_targets.clone()), @@ -493,6 +504,7 @@ fn run_checks<'py>( validation_refs: prepared.relation_refs.clone(), project_relations: prepared.project_relations.clone(), providers: providers.clone(), + target_config: Some(prepared.target_config.clone()), })?; record_run_state( invocation, @@ -673,10 +685,13 @@ pub(crate) fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P require_used_secrets(&discovered, &uses)?; let session: Bound<'_, PyAny> = call_with( &host_entry(py, UserCodeHost::BuildProviderSession)?, - &[( - "discovered_providers", - native_field(&invocation, "discovered_inputs")?.getattr("providers")?, - )], + &[ + ( + "discovered_providers", + native_field(&invocation, "discovered_inputs")?.getattr("providers")?, + ), + ("setup_context", prepared.target_config.setup_context(py)?), + ], )?; let outcome: PyResult = (|| { let results: Vec> = execute( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 1dce014f02..efc62cd85c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -1,5 +1,6 @@ //! Python check planning: run-state recording, dependency relations and read-side preflight. +use crate::bindings::_helpers::compile_driver::targets::TargetSettingsValues; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; @@ -263,6 +264,7 @@ pub(crate) struct ReadSideInputs<'py> { pub(crate) validation_refs: Bound<'py, PyAny>, pub(crate) project_relations: Bound<'py, PyAny>, pub(crate) providers: Bound<'py, PyAny>, + pub(crate) target_config: Option, } fn reference_kind(reference: &Bound<'_, PyAny>) -> PyResult { @@ -454,6 +456,13 @@ fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult ("target", native_field(&project, "effective_target_name")?), ("vars", native_field(&project, "effective_vars")?), ("is_reload", false.into_pyobject(py)?.to_owned().into_any()), + ( + "target_config", + match &inputs.target_config { + Some(settings) => settings.record(py)?, + None => py.None().into_bound(py), + }, + ), ( "default_database", inputs.adapter.call_method0("default_database")?, @@ -541,6 +550,7 @@ fn run_check_read_side_dependencies<'py>( validation_refs: arguments.value("validation_refs")?, project_relations: arguments.value("project_relations")?, providers: arguments.value("providers")?, + target_config: None, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index 7770b938b4..4bf39be9b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -36,10 +36,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { target::build_compile_target_context, module )?)?; - module.add_function(wrap_pyfunction!( - target::build_effective_target_namespace, - module - )?)?; module.add_function(wrap_pyfunction!( declaration_scope::build_declaration_scope, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 7bda60a51e..408b1c8897 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -13,8 +13,8 @@ use crate::bindings::_helpers::frontier::constants::{ }; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::compile_attachment::main::project_vars::project_vars; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; @@ -23,6 +23,8 @@ use sqlbuild_python_front::model_config::main::config::run_id::run_id; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; +/// Adapters whose database is a file, so a connection `database` is not a target location. +pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; /// The selected target's name and merged config, or `(None, None)`. pub(crate) fn resolved_target<'py>( @@ -150,35 +152,20 @@ pub(crate) fn connection_is_default_target(discovered: &Bound<'_, PyAny>) -> PyR Ok(true) } -/// Resolve the target and its effective database/schema without compiling resources. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_effective_target_namespace<'py>( - py: Python<'py>, - options: Option>, +/// The selected target's name, record, database and schema, resolved from `connection` and the adapter's defaults without compiling resources. +pub(crate) fn effective_target_namespace<'py>( + (discovered_inputs, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter_name, connection, adapter): (&str, &Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_effective_target_namespace", - options, - ( - &[ - "discovered_inputs", - "selected_target", - "connection_database", - "connection_schema", - "default_database", - "default_schema", - ], - &[], - ), - )?; - let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; - let selected_target: &Bound<'py, PyAny> = &arguments.value("selected_target")?; - let connection_database: Bound<'py, PyAny> = arguments.value("connection_database")?; - let connection_schema: Bound<'py, PyAny> = arguments.value("connection_schema")?; - let default_database: Bound<'py, PyAny> = arguments.value("default_database")?; - let default_schema: Bound<'py, PyAny> = arguments.value("default_schema")?; + let py: Python<'py> = adapter.py(); + let connection_database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name) { + py.None().into_bound(py) + } else { + mapping_get(connection, DATABASE_KEY)? + }; + let connection_schema: Bound<'py, PyAny> = mapping_get(connection, SCHEMA_KEY)?; + let default_database: Bound<'py, PyAny> = adapter.call_method0("default_database")?; + let default_schema: Bound<'py, PyAny> = adapter.call_method0("default_schema")?; let (target_name, target_config) = resolved_target(discovered_inputs, selected_target)?; let effective_vars: Bound<'py, PyAny> = project_vars(discovered_inputs, &target_config)?; let target_value = |key: &str| -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index e665d8bde7..f797a8508e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -2,10 +2,11 @@ use std::collections::{HashMap, HashSet}; +use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyString, PyStringMethods}; +use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::target_checks::constants::{ @@ -43,12 +44,16 @@ const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; use crate::bindings::_helpers::compile_attachment::target::resolved_target; use pyo3::IntoPyObjectExt; + +use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::{ErrorId, RecordId}; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; @@ -342,7 +347,10 @@ fn target_namespaces( let mut namespaces: Vec = Vec::new(); for source_entry in managed_sources { let namespace: Namespace = managed_source_namespace( - source_entry, + ( + native_field(source_entry, "database")?, + native_field(source_entry, "schema")?, + ), (&target_config, &connection, &effective_vars), adapter, )?; @@ -366,28 +374,28 @@ fn first_truthy<'py>( fallback() } -/// `_managed_source_namespace`: the database and schema a managed source loads into. -fn managed_source_namespace( - source_entry: &Bound<'_, PyAny>, +/// `_managed_source_namespace`: the database and schema a managed source with these own values loads into. +fn managed_source_namespace<'py>( + (source_database, source_schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), (target_config, connection, effective_vars): ( - &Bound<'_, PyAny>, - &Bound<'_, PyAny>, - &Bound<'_, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, ), - adapter: &Bound<'_, PyAny>, + adapter: &Bound<'py, PyAny>, ) -> PyResult { - let py: Python<'_> = source_entry.py(); - let database: Bound<'_, PyAny> = first_truthy( + let py: Python<'py> = adapter.py(); + let database: Bound<'py, PyAny> = first_truthy( &[ - native_field(source_entry, "database")?, + source_database, native_field(target_config, "database")?, mapping_get(connection, "database")?, ], || adapter.call_method0("default_database"), )?; - let schema: Bound<'_, PyAny> = first_truthy( + let schema: Bound<'py, PyAny> = first_truthy( &[ - native_field(source_entry, "schema")?, + source_schema, native_field(target_config, "loader_schema")?, native_field(target_config, "schema")?, mapping_get(connection, "schema")?, @@ -408,6 +416,136 @@ fn managed_source_namespace( Ok((part("database")?, part("schema")?)) } +/// The active target's locations as SQLBuild resolves them; `preserve` stays as written until `record`. +#[derive(Clone)] +pub(crate) struct TargetSettingsValues { + name: Option, + database: Option, + schema: Option, + loader_schema: Option, +} + +impl TargetSettingsValues { + /// The `TargetSettings` record: a `preserve` location reads `None` with its flag set. + pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { + let located = |value: &Option| -> PyResult> { + value + .as_deref() + .filter(|value| *value != PRESERVE_TARGET_VALUE) + .into_bound_py_any(py) + }; + let preserved = |value: &Option| -> Bound<'py, PyAny> { + PyBool::new(py, value.as_deref() == Some(PRESERVE_TARGET_VALUE)) + .to_owned() + .into_any() + }; + new_record_with( + py, + RecordId::TargetSettings, + &[ + ("name", self.name.clone().into_bound_py_any(py)?), + ("database", located(&self.database)?), + ("schema", located(&self.schema)?), + ("loader_schema", located(&self.loader_schema)?), + ("database_preserved", preserved(&self.database)), + ("schema_preserved", preserved(&self.schema)), + ], + ) + } + + /// The `ctx` a provider's `setup` receives in a command with these settings. + pub(crate) fn setup_context<'py>(&self, py: Python<'py>) -> PyResult> { + new_record_with( + py, + RecordId::ProviderSetupContext, + &[("target_config", self.record(py)?)], + ) + } +} + +/// The target `target_name` with its resolved locations; `loader_schema` is `None` when it resolves only inside a model, such as a `${CTX:...}` schema. +pub(crate) fn target_settings<'py>( + (discovered, target_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (Option, Option), + (connection, effective_vars, adapter): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult { + let py: Python<'py> = adapter.py(); + let target_config: Bound<'py, PyAny> = if target_name.is_none() { + new_record(py, RecordId::TargetConfig, (), None)? + } else { + resolve_target_config( + py, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + target_name, + )? + }; + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let loader_schema: Option = match managed_source_namespace( + (none.clone(), none), + (&target_config, connection, effective_vars), + adapter, + ) { + Ok((_, schema)) => schema, + Err(error) if error.is_instance_of::(py) => None, + Err(error) => return Err(error), + }; + Ok(TargetSettingsValues { + name: target_name.extract()?, + database, + schema, + loader_schema, + }) +} + +/// The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. +pub(crate) fn selected_target_settings<'py>( + (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter_name: &str, + (connection, effective_vars, adapter): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult { + let namespace: Bound<'py, PyTuple> = effective_target_namespace( + (discovered, selected_target), + (adapter_name, connection, adapter), + )?; + target_settings( + (discovered, &namespace.get_item(0)?), + ( + namespace.get_item(2)?.extract()?, + namespace.get_item(3)?.extract()?, + ), + (connection, effective_vars, adapter), + ) +} + +/// `target_settings` of a compiled project's target. +pub(crate) fn project_target_settings<'py>( + discovered: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + (connection, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult { + target_settings( + (discovered, &native_field(project, "effective_target_name")?), + ( + native_field(project, "effective_target_database")?.extract()?, + native_field(project, "effective_target_schema")?.extract()?, + ), + ( + connection, + &native_field(project, "effective_vars")?, + adapter, + ), + ) +} + /// A managed source's `database` and `schema` with project vars expanded and no context templates. pub(crate) fn expanded_loader_namespace<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 5be09d62b7..9abb15a730 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -33,11 +33,9 @@ use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehou use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; -use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; -use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; +use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; @@ -308,41 +306,14 @@ fn build_report( })?; let connection_config: Bound<'_, PyDict> = effective_connection(&context, (&adapter_name, &base_connection))?.cast_into()?; - let namespace_adapter: String = adapter_name.str()?.extract()?; - let namespaceless: bool = NAMESPACELESS_ADAPTERS.contains(&namespace_adapter.as_str()); - let namespace: Bound<'_, PyTuple> = build_effective_target_namespace( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", context.discovered_inputs.clone()), - ("selected_target", context.selected_target.clone()), - ( - "connection_database", - connection_config - .get_item("database")? - .filter(|_| !namespaceless) - .unwrap_or_else(|| py.None().into_bound(py)), - ), - ( - "connection_schema", - connection_config - .get_item("schema")? - .unwrap_or_else(|| py.None().into_bound(py)), - ), - ( - "default_database", - context.adapter.call_method0("default_database")?, - ), - ( - "default_schema", - context.adapter.call_method0("default_schema")?, - ), - ], - )?), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .cast_into()?; + let namespace: Bound<'_, PyTuple> = effective_target_namespace( + (&context.discovered_inputs, &context.selected_target), + ( + &adapter_name.str()?.extract::()?, + connection_config.as_any(), + &context.adapter, + ), + )?; let target_config: Bound<'_, PyAny> = namespace.get_item(1)?; let connection_name: Option = if target_config.is_none() { None diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 29afbc7e3f..fda90eed6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -16,9 +16,9 @@ use sqlbuild_cli::diff_report::types::PhaseSeconds; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; +use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use crate::bindings::_helpers::diff_report::support::{ - call, cli_error, error_text, none, object, progress_line, + call, cli_error, error_text, object, progress_line, }; use crate::bindings::_helpers::diff_report::tolerances::cli_tolerance_overrides; use sqlbuild_analysis::diff_selection::main::raw_query_shape::raw_query_shape; @@ -30,7 +30,6 @@ use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cl use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; @@ -38,7 +37,6 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_con use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; const GENERIC_DIALECT: &str = "generic"; -pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { cli_error( @@ -197,29 +195,11 @@ pub(crate) fn prepare_query_diff<'py>( ], )?), )?; - let database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name.as_str()) { - none(py) - } else { - mapping_get(&config, "database")? - }; - let namespace: Bound<'py, PyAny> = build_effective_target_namespace( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", selected_target.clone()), - ("connection_database", database.clone()), - ("connection_schema", mapping_get(&config, "schema")?), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ("default_schema", adapter.call_method0("default_schema")?), - ], - )?), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + let namespace: Bound<'py, PyAny> = effective_target_namespace( + (discovered, &selected_target), + (&adapter_name, &config, &adapter), + )? + .into_any(); let ResolvedNamespace { target_name, database, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index a1d82374fb..07b778128a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -22,6 +22,9 @@ use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionPro use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, selected_target_settings, +}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -79,6 +82,7 @@ struct Preparation<'py> { cursor_overrides: Bound<'py, PyAny>, provider_session: Bound<'py, PyAny>, project_relations: Bound<'py, PyAny>, + target_config: TargetSettingsValues, } fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { @@ -354,14 +358,23 @@ fn prepare<'py>( } else { new_record(py, RecordId::CursorOverrides, (), None)? }; + let target_config: TargetSettingsValues = selected_target_settings( + (discovered, &native_field(request, "selected_target")?), + &adapter_name.extract::()?, + (&connection_config, &runtime.1, &adapter), + )?; let provider_session: Bound<'py, PyAny> = call_with( &host_entry(py, UserCodeHost::BuildProviderSession)?, - &[( - "discovered_providers", - native_field(discovered, "providers")?, - )], + &[ + ( + "discovered_providers", + native_field(discovered, "providers")?, + ), + ("setup_context", target_config.setup_context(py)?), + ], )?; Ok(Preparation { + target_config, adapter_name: adapter_name.extract()?, project_relations: project_relations(invocation, &adapter)?, adapter, @@ -395,6 +408,7 @@ fn runtime_params<'py>( ("target", target_name.clone()), ("vars", effective_vars.clone()), ("is_reload", native_field(request, "reload")?), + ("target_config", preparation.target_config.record(py)?), ( "start_cursor_ts", cursor_timestamp(&native_field(overrides, "start_ts")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 87df77c561..bf73124167 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod orchestration_sensors; pub(crate) mod registration; pub(crate) mod resource_costs_state; +pub(crate) mod statement_report; pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs new file mode 100644 index 0000000000..f34809d85d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs @@ -0,0 +1,153 @@ +//! The warehouse statements of a `sqb test` or `sqb scenario test` run, with their compile time. + +use std::time::Instant; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTypeMethods}; +use sqlbuild_core::cost_allocation::main::statement_timings_sql::statement_timings_sql; +use sqlbuild_core::json::main::with_section::with_section; +use sqlbuild_core::progress::main::format_statement_summary::format_statement_summary; +use sqlbuild_core::progress::main::statement_summary_json::statement_summary_json; +use sqlbuild_core::progress::models::{StatementSummary, StatementTiming, StatementTimings}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +use crate::bindings::_helpers::cost_telemetry::values::cost_aware; + +/// The JSON document section the report adds. +const STATEMENTS_SECTION: &str = "statements"; +/// Query identifiers per query-history read. +const QUERY_ID_BATCH: usize = 1_000; +/// Extra query-history window before the run started, for statements issued while planning. +const LOOKBACK_MARGIN_SECONDS: u64 = 600; +/// Connection and statement limits of the history read, so it cannot stall a finished run. +const CONNECTION_LIMITS: [(&str, u64); 2] = [("login_timeout", 5), ("network_timeout", 15)]; +const STATEMENT_TIMEOUT_SECONDS: &str = "10"; + +/// The adapter and connection a run's statements went to, and when the run started. +pub(crate) struct StatementSource<'a, 'py> { + pub(crate) adapter_name: &'a str, + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) connection_config: &'a Bound<'py, PyAny>, + pub(crate) started: Instant, +} + +/// The statements this invocation finished, and what the warehouse reports about their time. +pub(crate) struct StatementReport { + summary: StatementSummary, + timing: StatementTiming, +} + +impl StatementReport { + /// The report of a `sqb test` invocation's adapter and connection. + pub(crate) fn of_invocation( + invocation: &Bound<'_, PyAny>, + started: Instant, + ) -> PyResult> { + Self::read( + invocation.py(), + &StatementSource { + adapter_name: &invocation.getattr("adapter_name")?.extract::()?, + adapter: &invocation.getattr("adapter")?, + connection_config: &invocation.getattr("connection_config")?, + started, + }, + ) + } + + /// The report of the installed projector, or nothing when no projector is installed. + pub(crate) fn read(py: Python<'_>, source: &StatementSource<'_, '_>) -> PyResult> { + let Some(projector) = current_projector(py)? else { + return Ok(None); + }; + let summary: StatementSummary = projector.get().statement_summary()?; + let timing: StatementTiming = statement_timing(py, source, &summary.query_ids)?; + Ok(Some(Self { summary, timing })) + } + + /// The summary line, without a trailing newline; nothing when no statements ran. + pub(crate) fn line(&self) -> Option { + format_statement_summary(&self.summary, &self.timing) + } + + /// `payload` with the `statements` section added. + pub(crate) fn with_section(&self, payload: &str) -> PyResult { + with_section( + payload, + STATEMENTS_SECTION, + statement_summary_json(&self.summary, &self.timing), + ) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string())) + } +} + +/// Compile and execution time from Snowflake query history; other adapters report none. +fn statement_timing( + py: Python<'_>, + source: &StatementSource<'_, '_>, + query_ids: &[String], +) -> PyResult { + let not_reported = || StatementTiming::NotReported(source.adapter_name.to_owned()); + if !cost_aware(source.adapter)? { + return Ok(not_reported()); + } + let capability: Bound<'_, PyAny> = source.adapter.call_method0("cost_capability")?; + let query_history: Bound<'_, PyAny> = + host_entry(py, ExecutorHost::CostCapability)?.getattr("SNOWFLAKE_QUERY_HISTORY")?; + if !capability.eq(query_history)? { + return Ok(not_reported()); + } + if query_ids.is_empty() { + return Ok(StatementTiming::Reported(StatementTimings::default())); + } + Ok(match read_timings(source, query_ids) { + Ok(timings) => StatementTiming::Reported(timings), + Err(error) if error.is_instance_of::(py) => StatementTiming::Failed(format!( + "query history read failed: {}", + error.get_type(py).name()? + )), + Err(error) => return Err(error), + }) +} + +fn read_timings( + source: &StatementSource<'_, '_>, + query_ids: &[String], +) -> PyResult { + let py: Python<'_> = source.adapter.py(); + let config: Bound<'_, PyDict> = source.connection_config.downcast::()?.copy()?; + let database: Option = match config.get_item("database")? { + Some(database) if !database.is_none() => Some(database.str()?.extract()?), + _ => None, + }; + for (name, limit) in CONNECTION_LIMITS { + config.set_item(name, limit)?; + } + let parameters: Bound<'_, PyDict> = PyDict::new(py); + parameters.set_item("STATEMENT_TIMEOUT_IN_SECONDS", STATEMENT_TIMEOUT_SECONDS)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("statement_params", parameters)?; + let lookback_seconds: u64 = source.started.elapsed().as_secs() + LOOKBACK_MARGIN_SECONDS; + let connection: Bound<'_, PyAny> = source.adapter.call_method1("connect", (config,))?; + let read = || -> PyResult { + let mut timings: StatementTimings = StatementTimings::default(); + for batch in query_ids.chunks(QUERY_ID_BATCH) { + let sql: String = statement_timings_sql(database.as_deref(), batch, lookback_seconds); + let (count, compile_ms, execution_ms): (u64, f64, f64) = connection + .call_method("execute", (sql,), Some(&options))? + .call_method0("fetchone")? + .extract()?; + timings.timed_statements += count; + timings.compile_ms += compile_ms; + timings.execution_ms += execution_ms; + } + Ok(timings) + }; + let timings: PyResult = read(); + let closed: PyResult> = source.adapter.call_method1("close", (&connection,)); + let timings: StatementTimings = timings?; + closed?; + Ok(timings) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 1422bd6b50..197f8556d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::change_selection::base_snapshot::{ use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, with_since_section, }; +use crate::bindings::_helpers::native_host::statement_report::{StatementReport, StatementSource}; use crate::bindings::_helpers::result_output::documents::format_scenario_execution_json_with; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::scenario_result_row; @@ -168,6 +169,18 @@ fn finish( if !failures.is_empty() { write(&run.stream, &format!("{failures}\n"))?; } + let report: Option = StatementReport::read( + py, + &StatementSource { + adapter_name: &run.adapter_name, + adapter: &run.adapter, + connection_config: &run.connection_config, + started, + }, + )?; + if let Some(line) = report.as_ref().and_then(StatementReport::line) { + write(&run.stream, &format!("{line}\n"))?; + } flush(&run.stream)?; write_completion(&run.stream, (&run.project.namespace, succeeded))?; let namespace: &ScenarioNamespace = &run.project.namespace; @@ -194,7 +207,11 @@ fn finish( )?), )?, run.since_json.as_deref(), - )?, + ) + .and_then(|payload| match &report { + Some(report) => report.with_section(&payload), + None => Ok(payload), + })?, ) .into_any(); write_execution_json_output( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 7b6a2236b4..91a8fbf00a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; use std::sync::Arc; +use std::time::Instant; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -41,6 +42,7 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pi use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::native_host::statement_report::StatementReport; use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; @@ -56,6 +58,7 @@ use crate::bindings::_helpers::sql_test_command::timeouts::{ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::emit::emit; use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -657,6 +660,7 @@ pub(crate) fn timeout_record<'py>( fn complete( invocation: &Bound<'_, PyAny>, (results, since_json): (&[SqlTestExecutionResult>], Option<&str>), + started: Instant, ) -> PyResult<(i32, String)> { let py: Python<'_> = invocation.py(); let mut failures: Vec = Vec::new(); @@ -676,18 +680,19 @@ fn complete( let style: CliStyle = CliStyle { use_color: invocation.getattr("use_color")?.is_truthy()?, }; - stream.call_method1( - "write", - (render_test_completion( - (passed, results.len()), - &failures, - style, - ),), - )?; - stream.call_method0("flush")?; + let completion: String = render_test_completion((passed, results.len()), &failures, style); + emit(&stream, &completion)?; + let report: Option = StatementReport::of_invocation(invocation, started)?; + if let Some(line) = report.as_ref().and_then(StatementReport::line) { + emit(&stream, &format!("\n{line}\n"))?; + } + let payload: String = with_since_section(test_execution_json(py, results)?, since_json)?; Ok(( test_exit_code(passed, results.len()), - with_since_section(test_execution_json(py, results)?, since_json)?, + match report { + Some(report) => report.with_section(&payload)?, + None => payload, + }, )) } @@ -801,6 +806,7 @@ fn run_sql_tests<'py>( (modules, resolve): (Option>, bool), ) -> PyResult<(i32, Vec, Option)> { let py: Python<'py> = request.py(); + let started: Instant = Instant::now(); let inspect_only: bool = native_field(request, "inspect")?.is_truthy()?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; let project_dir: Bound<'_, PyAny> = native_field(invocation, "effective_project_dir")?; @@ -895,7 +901,8 @@ fn run_sql_tests<'py>( &pipeline.getattr("plan_output")?, &results, )?; - let (code, payload): (i32, String) = complete(invocation, (&results, since_json.as_deref()))?; + let (code, payload): (i32, String) = + complete(invocation, (&results, since_json.as_deref()), started)?; Ok((code, python_tests, Some(payload))) } diff --git a/fensu.toml b/fensu.toml index 0b2252bb2e..c4f6c000b9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1670,7 +1670,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147491 } +thresholds = { max_crate_code_lines = 147789 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -2384,7 +2384,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=30", @@ -2474,11 +2474,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", @@ -2729,7 +2729,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=99", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=97", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 9d449d32ef..c3f598dc52 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -125,10 +125,12 @@ daily_revenue_minimal PASS 0.84s expect assertion no_negative_revenue PASS PASS=1 FAIL=0 TOTAL=1 (0.91s) +Statements: 9 (CREATE 3, SELECT 3, DROP 2, ALTER 1); compile time not reported by duckdb ``` Each scenario row shows its wall time and the summary shows the total. `--json` reports the same -values as `scenarios[].duration_ms` and `summary.duration_ms`. +values as `scenarios[].duration_ms` and `summary.duration_ms`. The `Statements:` line and the JSON +`statements` section are described in [Statement summary](test.md#statement-summary). Local scenarios add ERROR and SKIP statuses: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 59804b6c8d..8ee38ea28f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/cli/test/ - Usage - Flags +- Statement summary - Python tests - Run only affected tests - Examples @@ -64,6 +65,31 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Statement summary + +After the pass/fail summary, `sqb test` and `sqb scenario test` print how many warehouse statements +the SQL tests or scenarios sent, by leading keyword, and how long the warehouse spent compiling and executing them: + +``` +Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s +``` + +With `--json`, the document gains a `statements` section with `total`, `by_kind` and +`compile_time`. + +| Adapter | Compile and execution time | +| --- | --- | +| Snowflake | Read from `INFORMATION_SCHEMA.QUERY_HISTORY` after the run; `compile_time` has `compile_ms`, `execution_ms`, `timed_statements` and `identified_statements` | +| Other adapters | Not reported; the line says `compile time not reported by ` and `compile_time` is `null` | + +Details: + +- On Snowflake, statements that query history does not return yet are left out of the times, and + the line says how many were found, such as `(10 of 12 statements in query history)`. +- When the query-history read fails, the counts still print and the line says + `compile time unavailable (...)`; the run's exit code does not change. +- Statements from [Python tests](#python-tests) are not counted. + ## Python tests Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md index 62f69e0b13..d2ad1a345f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md @@ -125,6 +125,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o | `ctx.hook_index` | Zero-based position in the authored hook list | | `ctx.run_id` | Current run ID | | `ctx.target` | Active target | +| `ctx.target_config` | The active target's database and schemas ([Target settings](../../python-nodes/overview.md#target-settings)) | | `ctx.vars` | Effective project variables | | `ctx.destination` | Destination relation metadata | | `ctx.relation(ref)` | Adapter-qualified relation for a [declared read](#declared-reads) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index 9d4aca5f7e..94e291ab16 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -254,6 +254,7 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `current_cursor_value` | `object \| None` | Current `MAX(cursor_column)` from the destination, or `None` if the table does not exist or has no cursor column | | `run_id` | `str` | Unique identifier for this execution run | | `target` | `str \| None` | Active target name (e.g. `dev`, `prod`) | +| `target_config` | `TargetSettings` | The active target's database, schema and loader schema ([Target settings](overview.md#target-settings)) | | `vars` | `dict` | Project variables (merged from project, target, and local config) | | `is_reload` | `bool` | `True` when `--reload` was passed | | `start_cursor_ts` | `datetime \| None` | Timestamp cursor start override from `--start-cursor-ts` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 1488233602..bd1768654e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -113,6 +113,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset |--------|-------------| | `ctx.run_id` | Unique identifier for this run | | `ctx.target` | Active target name | +| `ctx.target_config` | The active target's database and schemas, as SQLBuild resolves them (see [Target settings](#target-settings)) | | `ctx.vars` | Project variables | | `ctx.is_reload` | `True` when `--reload` was passed | | `ctx.adapter` / `ctx.connection` | Adapter and live connection | @@ -128,6 +129,30 @@ Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check context Providers can also be injected directly as function parameters by name. See [Providers](providers.md) for details. +### Target settings + +Read the active target's locations from `ctx.target_config` instead of keeping each target's schema names in Python. The values are resolved the way SQLBuild resolves them for models and loaders, including `sqlbuild_local.toml` overrides, and are read-only: + +| Field | Value | +|-------|-------| +| `name` | Active target name, or `None` without targets | +| `database` | Database models build into, or `None` when the adapter has none or the value is `preserve` | +| `schema` | Schema models build into, or `None` when it is `preserve` | +| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema`; `None` when that schema uses a `${CTX:...}` template | +| `database_preserved` / `schema_preserved` | `True` when the target sets that value to `preserve` | + +```python +@asset +def partner_export(ctx): + """Export orders to the partner staging table.""" + target = ctx.target_config + staging = f"{target.database}.{target.loader_schema}.partner_staging" + ctx.execute_sql(f"CREATE OR REPLACE TABLE {staging} AS SELECT * FROM orders") + return ctx.result() +``` + +Connection settings and secrets are not part of `ctx.target_config`. + ## Returns and skips Tasks and assets return through `ctx.result(...)`: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index 827a3b7360..c9217cb9a4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -171,6 +171,23 @@ class WarehouseClient(Provider): Both `setup` and `teardown` are optional. A provider with only field declarations and no lifecycle methods is valid; it acts as a validated configuration object. +### The setup context + +In `sqb build`, `sqb check` and `sqb load`, `setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](overview.md#target-settings)): + +```python +class PartnerStage(Provider): + """Stage for partner exports.""" + + def setup(self, ctx): + target = ctx.target_config + self.stage = f"{target.database}.{target.loader_schema}.partner_stage" +``` + +Details: + +- A provider first used by an event exporter or command output sink, or by an orchestration sensor or calendar, is set up with `ctx` set to `None`. Check for `None` when such a provider also reads the target. + ## Configuration from environment variables Because `Provider` extends `pydantic-settings BaseSettings`, fields without defaults are read from environment variables. The environment variable name matches the field name in uppercase: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d6d38207f2..11632aced0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -661,6 +661,12 @@ class NativeSqlTestArtifact(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> NativeSqlTestArtifact: ... +class ProviderSetupContext(NativeRecord): + """What a provider's setup receives: the active target's resolved settings.""" + + target_config: TargetSettings | None + def __init__(self, target_config: TargetSettings | None = ...) -> None: ... + class RelationFixtureDiagnostic(NativeRecord): """One statically provable invalid fixture condition.""" @@ -11900,6 +11906,25 @@ class TargetDiffConfig(NativeRecord): max_full_rows: int | Literal["unlimited"] | None = ..., ) -> None: ... +class TargetSettings(NativeRecord): + """The active target's database and schemas, resolved as SQLBuild uses them.""" + + name: str | None + database: str | None + schema: str | None + loader_schema: str | None + database_preserved: bool + schema_preserved: bool + def __init__( + self, + name: str | None = ..., + database: str | None = ..., + schema: str | None = ..., + loader_schema: str | None = ..., + database_preserved: bool = ..., + schema_preserved: bool = ..., + ) -> None: ... + class TargetWarehousesConfig(NativeRecord): """Optional per-command-group default warehouses for one target.""" diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 18f57fbb91..727e48f49e 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -24,7 +24,6 @@ from sqlbuild.executor.build.constants import ( BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, BUILD_SOURCE_FRESHNESS_BLOCKED_CODE, - BUILD_WORKER_FAILED_CODE, INCREMENTAL_ACTIONS, ) from sqlbuild.executor.build.models import ( @@ -65,20 +64,6 @@ def _build_worker_success_completion( return NodeCompletion(key=key, result=result) -def _build_worker_failure_completion( - *, key: CompiledObjectKey, error: BaseException -) -> NodeCompletion: - return NodeCompletion( - key=key, - result=ModelExecutionResult( - model_name=key.name, - status=ExecutionStatus.FAILED, - error_code=BUILD_WORKER_FAILED_CODE, - error_message=str(error), - ), - ) - - def _dispatch_model( *, context: ModelMaterializationContext, diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 84ae9ffec6..146df17c03 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -72,6 +72,7 @@ def execute_build_source_node( runtime=LoadRuntimeParams( run_id=runtime.run_id, target=runtime.target, + target_config=runtime.target_config, vars=runtime.effective_vars or {}, is_reload=runtime.loader_is_reload, runtime_dir=runtime.runtime_dir, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 5f55987d43..e9b67b8169 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -64,7 +64,6 @@ reconcile_model_retention, ) from sqlbuild.executor.build._helpers.scheduler import ( - _build_worker_failure_completion, _build_worker_success_completion, _dispatch_model, _model_execution_resource_kind, @@ -453,6 +452,7 @@ def _run_serial(self) -> None: self._connection_pool.put(connection) def _run_concurrent(self) -> None: + escaped: BaseException | None = None with ThreadPoolExecutor(max_workers=self._max_concurrency) as pool: self._report_scheduler_state() while self._ready or self._in_flight: @@ -518,8 +518,16 @@ def _run_concurrent(self) -> None: completion: NodeCompletion = self._completion_queue.get() self._in_flight.discard(completion.key) - self._handle_completion(key=completion.key, result=completion.result) + if completion.error is not None: + if escaped is None: + escaped = completion.error + self._stop = True + continue + if completion.result is not None: + self._handle_completion(key=completion.key, result=completion.result) self._report_scheduler_state() + if escaped is not None: + raise escaped def _report_scheduler_state(self) -> None: callback: Callable[[SchedulerState], None] | None = self._callbacks.on_scheduler_state @@ -548,7 +556,7 @@ def _worker(self, key: CompiledObjectKey) -> None: completion_queue=self._completion_queue, execute=lambda connection: self._execute_node(key=key, connection=connection), build_success=_build_worker_success_completion, - build_failure=_build_worker_failure_completion, + build_failure=NodeCompletion, ) def _is_concurrent_microbatch(self, key: CompiledObjectKey) -> bool: @@ -575,7 +583,7 @@ def _concurrent_microbatch_worker( key=key, pool=pool, connection=connection ), build_success=_build_worker_success_completion, - build_failure=_build_worker_failure_completion, + build_failure=NodeCompletion, ) def _execute_concurrent_microbatch_node( @@ -1210,6 +1218,7 @@ def _execute_model_node( hook_functions=self._plan.hook_functions, enforce_explicit_references=self._plan.enforce_explicit_references, effective_target_name=self._target, + target_config=self._runtime.target_config, effective_vars=self._effective_vars, providers=self._providers, python_identity_recorder=self._python_identity_recorder, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index ece162eb01..5b9521edf0 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -18,6 +18,7 @@ SnapshotsConfig, SourceLoadPlanEntry, SqlTestPlanEntry, + TargetSettings, ) from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode @@ -78,6 +79,7 @@ class BuildRuntimeParams: start_cursor_int: int | None = None end_cursor_int: int | None = None target: str = "" + target_config: TargetSettings | None = None effective_vars: dict[str, object] | None = None use_color: bool = False providers: ProviderContainer | None = None @@ -196,7 +198,7 @@ class BuildIndexes: @dataclass(frozen=True) class NodeCompletion: - """Result of a single node execution passed back from a worker to the scheduler.""" + """A node's result, or the error that escaped it, passed back from a worker.""" key: CompiledObjectKey result: ( @@ -205,7 +207,9 @@ class NodeCompletion: | FunctionExecutionResult | SqlTestExecutionResult | LoadExecutionResult - ) + | None + ) = None + error: BaseException | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/load/_helpers/external.py b/src/sqlbuild/executor/load/_helpers/external.py index b0f4773988..c85d333541 100644 --- a/src/sqlbuild/executor/load/_helpers/external.py +++ b/src/sqlbuild/executor/load/_helpers/external.py @@ -56,6 +56,7 @@ def execute_external_source_load( run_id=runtime.run_id, runtime_dir=runtime.runtime_dir, target=runtime.target, + target_config=runtime.target_config, vars=runtime.vars, is_reload=runtime.is_reload, use_color=runtime.use_color, diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index d287c8681c..3ee4a427b4 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -94,6 +94,7 @@ def build_loader_context( run_id=runtime.run_id, runtime_dir=runtime.runtime_dir, target=runtime.target, + target_config=runtime.target_config, vars=runtime.vars, is_reload=runtime.is_reload, use_color=runtime.use_color, diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 857aff8e6f..38f6e970b0 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry, TargetSettings from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent @@ -149,6 +149,7 @@ class LoaderContext(DependencyResultReader): runtime_dir: Path = Path("target") on_progress: Callable[[str], None] | None = None relation_guard: RuntimeRelationGuard | None = field(default=None, repr=False) + target_config: TargetSettings | None = None def execute_sql(self, sql: str) -> Any: if self.relation_guard is not None: @@ -281,6 +282,7 @@ class LoadRuntimeParams: target: str | None vars: dict[str, object] is_reload: bool + target_config: TargetSettings | None = None runtime_dir: Path = Path("target") start_cursor_ts: datetime | None = None end_cursor_ts: datetime | None = None diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index dafc8e882a..194ea2ca61 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -422,6 +422,7 @@ def _build_context( statement_recorder=statement_recorder, run_state=run_state, result_store=result_store, + target_config=runtime.target_config, default_database=runtime.default_database, default_schema=runtime.default_schema, relation_targets=runtime.resolved_relation_targets, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 6a24e47e14..d364544c48 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -283,6 +283,7 @@ def _execute_ingress_loader( runtime=LoadRuntimeParams( run_id=runtime.run_id, target=runtime.target, + target_config=runtime.target_config, vars=runtime.vars, is_reload=runtime.is_reload, start_cursor_ts=runtime.start_cursor_ts, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 220af44090..508a4b4d03 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -165,6 +165,7 @@ def _execute_one_python_check( statement_recorder=StatementRecorder(), run_state=run_state, result_store=result_store, + target_config=runtime.target_config, default_database=runtime.default_database, default_schema=runtime.default_schema, relation_targets=runtime.resolved_relation_targets, diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 4d7ffae694..661af2edc7 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -8,6 +8,7 @@ from datetime import datetime from typing import Any +from sqlbuild._native import TargetSettings from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( @@ -134,6 +135,7 @@ class PythonNodeRuntime: target: str | None vars: dict[str, object] is_reload: bool + target_config: TargetSettings | None = None default_database: str | None = None default_schema: str | None = None relation_targets: dict[SqlResourceRef, str] | None = None @@ -178,6 +180,7 @@ class BasePythonNodeContext(DependencyResultReader): statement_recorder: StatementRecorder run_state: PythonNodeRunState | None = None result_store: Any | None = None + target_config: TargetSettings | None = None default_database: str | None = None default_schema: str | None = None relation_targets: dict[SqlResourceRef, str] = field(default_factory=dict) diff --git a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py index 6841a6b8f4..3cf0b6b512 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py @@ -132,6 +132,7 @@ def build_model_hook_run( destination=context.entry.destination, run_id=context.run_id, target=context.effective_target_name, + target_config=context.target_config, effective_vars=context.effective_vars if effective_vars is None else effective_vars, statement_recorder=statement_recorder, providers=context.providers, diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index a83dee9f83..5974507a52 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -466,6 +466,7 @@ def build_hook_context( # noqa: PLR0913 hook_index=invocation.index, run_id=hook_run.run_id, target=hook_run.target, + target_config=hook_run.target_config, vars=hook_run.effective_vars if hook_run.effective_vars is not None else {}, destination=relation, adapter_name=adapter.adapter_name, diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 551539f588..674a5fba62 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -26,6 +26,7 @@ PythonHookEntry, SourceEntry, StartCursorsConfig, + TargetSettings, TimestampValue, ) from sqlbuild._native import cursor_compare as compare @@ -104,6 +105,7 @@ class HookContext: providers: ProviderContainer = field(default_factory=_empty_provider_container, repr=False) relations: Mapping[SqlResourceRef, str] = field(default_factory=dict, repr=False) relation_guard: RuntimeRelationGuard | None = field(default=None, repr=False) + target_config: TargetSettings | None = None def execute_sql(self, sql: str) -> None: if self.relation_guard is not None: @@ -314,6 +316,7 @@ class ModelMaterializationContext: schema_prepared: bool = False hook_functions: tuple[DiscoveredHookFunction, ...] = () effective_target_name: str | None = None + target_config: TargetSettings | None = None effective_vars: Mapping[str, object] | None = None providers: ProviderContainer | None = None python_identity_recorder: PythonIdentityRecorder | None = None @@ -532,6 +535,7 @@ class HookRunContext: destination: CompiledRelationLocation | None = None run_id: str = "" target: str | None = None + target_config: TargetSettings | None = None effective_vars: Mapping[str, object] | None = None statement_recorder: StatementRecorder | None = None providers: ProviderContainer | None = None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 56620025a9..b20aad918f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1632,3 +1632,23 @@ class PythonUpstreamResultE2ETestCase: select: str upstream_select: str expected_error: str + + +@dataclass(frozen=True) +class PythonTargetConfigBuildE2ETestCase: + """Test case for the active target's settings seen by Python nodes and loaders.""" + + description: str + project_target_lines: str + local_target_lines: str + expected_settings: dict[str, object] + + +@dataclass(frozen=True) +class CtxTargetSchemaE2ETestCase: + """Test case for commands under a target schema that resolves only inside a model.""" + + description: str + target_schema: str + expected_model_schema: str + expected_loader_schema: str | None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index d788ead43c..e9178ec58a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -13,6 +13,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( BuildE2ETestCase, + CtxTargetSchemaE2ETestCase, DeferCloneBuildE2ETestCase, DirectChangesOnlyBuildE2ETestCase, DirectPythonBuildHardeningE2ETestCase, @@ -20,6 +21,7 @@ PythonLoaderPersistedResultBuildE2ETestCase, PythonLoaderStatusResultBuildE2ETestCase, PythonPersistedResultBuildE2ETestCase, + PythonTargetConfigBuildE2ETestCase, PythonTargetIsolationBuildE2ETestCase, SelectionAwareStalenessBuildE2ETestCase, ) @@ -36,6 +38,15 @@ table_exists, ) +_TARGET_CONFIG_RECORDER: str = ( + "import dataclasses, json\n" + "from pathlib import Path\n\n" + "def record(ctx, node):\n" + " line = {'node': node, **dataclasses.asdict(ctx.target_config)}\n" + " with Path(__file__).parents[1].joinpath('target_config.jsonl').open('a') as out:\n" + " out.write(json.dumps(line) + '\\n')\n" +) + @pytest.mark.parametrize( "test_case", @@ -1808,6 +1819,212 @@ def test_given_loader_status_result_when_building_then_persists_loader_status_ro ) +@pytest.mark.parametrize( + "test_case", + [ + PythonTargetConfigBuildE2ETestCase( + description="local override of schema and loader_schema", + project_target_lines='schema = "dev"\nloader_schema = "raw_dev"\n', + local_target_lines='schema = "dev_local"\nloader_schema = "raw_local"\n', + expected_settings={ + "name": "dev", + "database": None, + "schema": "dev_local", + "loader_schema": "raw_local", + "database_preserved": False, + "schema_preserved": False, + }, + ), + PythonTargetConfigBuildE2ETestCase( + description="loader_schema falls back to the target schema", + project_target_lines='schema = "dev"\n', + local_target_lines='schema = "dev_local"\n', + expected_settings={ + "name": "dev", + "database": None, + "schema": "dev_local", + "loader_schema": "dev_local", + "database_preserved": False, + "schema_preserved": False, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_local_target_override_when_running_python_then_every_context_sees_resolved_target( + test_case: PythonTargetConfigBuildE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_target_config_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "python_target_config_project"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "python_target_config_project.duckdb"\n\n' + "[targets.dev]\n" + test_case.project_target_lines + ), + "python/recorder.py": _TARGET_CONFIG_RECORDER, + "providers/target_probe.py": ( + "from python.recorder import record\n" + "from sqlbuild.providers import Provider\n\n" + "class TargetProbe(Provider):\n" + " '''Test provider TargetProbe.'''\n\n" + " def setup(self, ctx):\n" + " record(ctx, 'provider')\n" + ), + "python/tasks/configured.py": ( + "from providers.target_probe import TargetProbe\n" + "from python.recorder import record\n" + "from sqlbuild.tasks import task\n\n" + "@task\n" + "def configured_task(ctx, target_probe: TargetProbe):\n" + " '''Test task configured_task.'''\n" + " record(ctx, 'task')\n" + " return ctx.result()\n" + ), + "python/loaders/raw.py": ( + "from providers.target_probe import TargetProbe\n" + "from python.recorder import record\n" + "from sqlbuild.loaders import loader\n\n" + "@loader\n" + "def raw_orders(ctx, target_probe: TargetProbe):\n" + " '''Test loader raw_orders.'''\n" + " record(ctx, 'loader')\n" + " return [{'order_id': 1}]\n" + ), + "python/checks/configured.py": ( + "from providers.target_probe import TargetProbe\n" + "from python.recorder import record\n" + "from python.tasks.configured import configured_task\n" + "from sqlbuild.checks import check\n\n" + "@check(depends_on=configured_task)\n" + "def configured_check(ctx, target_probe: TargetProbe):\n" + " '''Test check configured_check.'''\n" + " record(ctx, 'check')\n" + " return ctx.pass_(message='recorded')\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " managed: true\n" + " write_strategy: table\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + ), + }, + ) + (project_dir / "sqlbuild_local.toml").write_text( + "[targets.dev]\n" + test_case.local_target_lines, encoding="utf-8" + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("--no-color", *command), project_dir=project_dir) + for command in ( + ("build", "--select", "configured_task", "raw_orders"), + ("load", "--select", "raw_orders"), + ("check", "--select", "configured_check"), + ) + ] + + for result in results: + assert result.returncode == 0, result.stdout + result.stderr + recorded: list[dict[str, object]] = [ + json.loads(line) + for line in (project_dir / "target_config.jsonl").read_text(encoding="utf-8").splitlines() + ] + # Each command (build, load, check) sets the provider up once, before its first node runs. + assert [str(line.pop("node")) for line in recorded] == [ + "provider", + "task", + "loader", + "check", + "provider", + "loader", + "provider", + "check", + ] + assert all(line == test_case.expected_settings for line in recorded), recorded + + +@pytest.mark.parametrize( + "test_case", + [ + CtxTargetSchemaE2ETestCase( + description="a per-model schema has no loader location and fails no command", + target_schema="s_${CTX:model.name}", + expected_model_schema="s_orders", + expected_loader_schema=None, + ) + ], + ids=lambda case: case.description, +) +def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_loader_schema( + test_case: CtxTargetSchemaE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="ctx_target_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "ctx_target_project"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "ctx_target_project.duckdb"\n\n' + "[targets.dev]\n" + f'schema = "{test_case.target_schema}"\n' + ), + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", + "python/recorder.py": _TARGET_CONFIG_RECORDER, + "python/loaders/raw.py": ( + "from python.recorder import record\n" + "from sqlbuild.loaders import loader\n\n" + "@loader\n" + "def raw_orders(ctx):\n" + ' """Test loader raw_orders."""\n' + " record(ctx, 'loader')\n" + " return [{'order_id': 1}]\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " managed: true\n" + " schema: raw\n" + " write_strategy: table\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + ), + }, + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("--no-color", *command), project_dir=project_dir) + for command in (("build", "--select", "orders"), ("load",), ("check",)) + ] + + for result in results: + assert result.returncode == 0, result.stdout + result.stderr + db_path: Path = project_dir / "ctx_target_project.duckdb" + assert table_exists( + db_path=db_path, table_name="orders", schema=test_case.expected_model_schema + ) + assert table_exists(db_path=db_path, table_name="raw_orders", schema="raw") + recorded: list[dict[str, object]] = [ + json.loads(line) + for line in (project_dir / "target_config.jsonl").read_text(encoding="utf-8").splitlines() + ] + assert [(line["node"], line["name"], line["loader_schema"]) for line in recorded] == [ + ("loader", "dev", test_case.expected_loader_schema) + ], recorded + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 9f8e2bd4af..1a1e72bcf6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -638,3 +638,23 @@ class PythonTestsJsonE2ETestCase: expected_status: str expected_python_status: str expected_python_files: tuple[str, ...] + + +@dataclass(frozen=True) +class StatementSummaryE2ETestCase: + """A `sqb test` run whose statement summary must count each finished statement once.""" + + description: str + expected_doubled: tuple[int, ...] + expected_exit_code: int + expected_by_kind: Mapping[str, int] + expected_line: str + + +@dataclass(frozen=True) +class ScenarioStatementSummaryE2ETestCase: + """A `sqb scenario test --json` run whose statement summary must match its progress line.""" + + description: str + expected_exit_code: int + expected_kinds: frozenset[str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index eea08cac2f..bfa49fbea8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -2477,3 +2477,13 @@ def prepare_since_repository(*, tmp_path: Path, extra_files: Mapping[str, str]) ) commit_project_on_main(repository=project) return project + + +def statement_summary_test_file(name: str, doubled: int) -> str: + """Return a SQL test of the `orders` model expecting `doubled` for an amount of 5.""" + + return ( + f'TEST (name "{name}");\n\nWITH\n' + "__source__raw_orders AS (\n SELECT 1 AS order_id, 5 AS amount\n),\n" + f"__expected__orders AS (\n SELECT 1 AS order_id, {doubled} AS doubled\n)\nSELECT 1\n" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py new file mode 100644 index 0000000000..3c3da37a76 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py @@ -0,0 +1,134 @@ +"""`sqb test` and `sqb scenario test` report the warehouse statements they ran, by kind.""" + +import json +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( + build_scenario_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + ScenarioStatementSummaryE2ETestCase, + StatementSummaryE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import statement_summary_test_file +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: amount\n type: INTEGER\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n\n" + 'SELECT order_id, amount * 2 AS doubled FROM __source("raw_orders")\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + ( + # Each test of one real model stores the model's output (CREATE), compares it with the + # expected rows (WITH) and drops it (DROP). + StatementSummaryE2ETestCase( + description="two passing tests count three statements each", + expected_doubled=(10, 10), + expected_exit_code=0, + expected_by_kind={"CREATE": 2, "DROP": 2, "WITH": 2}, + expected_line=( + "Statements: 6 (CREATE 2, DROP 2, WITH 2); compile time not reported by duckdb" + ), + ), + # A failed comparison also reads its unexpected and missing sample rows (two more WITH). + StatementSummaryE2ETestCase( + description="a failing test's statements count too", + expected_doubled=(10, 99), + expected_exit_code=1, + expected_by_kind={"CREATE": 2, "DROP": 2, "WITH": 4}, + expected_line=( + "Statements: 8 (WITH 4, CREATE 2, DROP 2); compile time not reported by duckdb" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_sql_tests_when_running_then_statement_summary_counts_each_statement( + tmp_path: Path, test_case: StatementSummaryE2ETestCase +) -> None: + files: dict[str, str] = dict(_PROJECT_FILES) + for index, doubled in enumerate(test_case.expected_doubled): + files[f"tests/unit/test_orders_{index}.sql"] = statement_summary_test_file( + f"orders_{index}", doubled + ) + project: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="statement_summary", repo_files=files + ) + + text_run: CompletedProcess[str] = run_sqb(command=("--no-color", "test"), project_dir=project) + json_run: CompletedProcess[str] = run_sqb( + command=("--no-color", "test", "--json"), project_dir=project + ) + + assert text_run.returncode == test_case.expected_exit_code, text_run.stdout + text_run.stderr + assert test_case.expected_line in text_run.stdout.splitlines(), text_run.stdout + assert json_run.returncode == test_case.expected_exit_code, json_run.stderr + assert json.loads(json_run.stdout)["statements"] == { + "total": sum(test_case.expected_by_kind.values()), + "by_kind": dict(test_case.expected_by_kind), + "compile_time": None, + } + assert test_case.expected_line in json_run.stderr.splitlines(), json_run.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + ScenarioStatementSummaryE2ETestCase( + description="passing and failing scenarios report their statements", + expected_exit_code=1, + expected_kinds=frozenset({"CREATE", "DROP"}), + ), + ), + ids=lambda case: case.description, +) +def test_given_scenarios_when_running_with_json_then_statement_summary_matches_progress_line( + tmp_path: Path, test_case: ScenarioStatementSummaryE2ETestCase +) -> None: + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="scenario_statements", + repo_files=build_scenario_project_files(), + ) + + result: CompletedProcess[str] = run_sqb( + command=("--no-color", "scenario", "test", "--json"), project_dir=project + ) + + assert result.returncode == test_case.expected_exit_code, result.stderr + statements: dict[str, object] = json.loads(result.stdout)["statements"] + by_kind: dict[str, int] = statements["by_kind"] # type: ignore[assignment] + assert statements["total"] == sum(by_kind.values()), statements + assert test_case.expected_kinds <= set(by_kind), statements + assert statements["compile_time"] is None + kinds: str = ", ".join( + f"{kind} {count}" + for kind, count in sorted(by_kind.items(), key=lambda item: (-item[1], item[0])) + ) + assert ( + f"Statements: {statements['total']} ({kinds}); compile time not reported by duckdb" + in result.stderr.splitlines() + ), result.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index c480609117..78536260c1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -111,7 +111,7 @@ class WorkerExecutionTestCase: model_sql: str warehouse_query: str expected_status: str - expected_rows: list[tuple[int]] + expected_rows: list[tuple[int] | tuple[str]] expected_node_count: int = 1 # Node counts by status when the nodes don't all share the run's status. expected_node_counts: dict[str, int] | None = None diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 4b469b733a..344395a439 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -1454,6 +1454,64 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "a rejected start report stops the run at concurrency 2", + "SELECT count(*) AS amount FROM range(500000000) t(value) WHERE value % 7 = 0", + "SELECT table_name FROM duckdb_tables() " + "WHERE table_name IN ('orders', 'refunds', 'returns') ORDER BY table_name", + "partial", + [("orders",)], + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\nconcurrency = 2\n' + ), + "models/refunds.sql": ( + "MODEL (description 'Refunds.', materialized table);\nSELECT 2 AS amount\n" + ), + "models/returns.sql": ( + "MODEL (description 'Returns.', materialized table);\n" + 'SELECT amount FROM __ref("orders")\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders, refunds and returns."\n' + 'select = "orders refunds returns"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + ), + }, + store_setup=( + "CREATE TRIGGER test_reject_refunds_start BEFORE UPDATE OF started_at " + "ON sqb_run_nodes WHEN NEW.node_key = 'model:refunds' " + "AND NEW.started_at IS NOT NULL BEGIN " + "SELECT RAISE(ABORT, 'synthetic rejection of the refunds start'); END;\n" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_rejected_node_report_when_concurrent_then_the_run_stops_before_later_nodes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + nodes: list[tuple[str, str, bool]] = control.execute( + "SELECT node_key, status, started_at IS NOT NULL FROM sqb_run_nodes " + "WHERE run_id=? AND kind='node' ORDER BY node_key", + (run["run_id"],), + ).fetchall() + assert ("model:returns", "skipped", False) in nodes, (nodes, worker.log.read_text()) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index 43e2c46638..922bc1c2d4 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -118,10 +118,12 @@ daily_revenue_minimal PASS 0.84s expect assertion no_negative_revenue PASS PASS=1 FAIL=0 TOTAL=1 (0.91s) +Statements: 9 (CREATE 3, SELECT 3, DROP 2, ALTER 1); compile time not reported by duckdb ``` Each scenario row shows its wall time and the summary shows the total. `--json` reports the same -values as `scenarios[].duration_ms` and `summary.duration_ms`. +values as `scenarios[].duration_ms` and `summary.duration_ms`. The `Statements:` line and the JSON +`statements` section are described in [Statement summary](/docs/cli/test#statement-summary). Local scenarios add ERROR and SKIP statuses: diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index 67cfd4fd04..fb168a62e5 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -55,6 +55,31 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Statement summary + +After the pass/fail summary, `sqb test` and `sqb scenario test` print how many warehouse statements +the SQL tests or scenarios sent, by leading keyword, and how long the warehouse spent compiling and executing them: + +``` +Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s +``` + +With `--json`, the document gains a `statements` section with `total`, `by_kind` and +`compile_time`. + +| Adapter | Compile and execution time | +| --- | --- | +| Snowflake | Read from `INFORMATION_SCHEMA.QUERY_HISTORY` after the run; `compile_time` has `compile_ms`, `execution_ms`, `timed_statements` and `identified_statements` | +| Other adapters | Not reported; the line says `compile time not reported by ` and `compile_time` is `null` | + +Details: + +- On Snowflake, statements that query history does not return yet are left out of the times, and + the line says how many were found, such as `(10 of 12 statements in query history)`. +- When the query-history read fails, the counts still print and the line says + `compile time unavailable (...)`; the run's exit code does not change. +- Statements from [Python tests](#python-tests) are not counted. + ## Python tests Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx index 9bac6070ee..009f5e7f13 100644 --- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx @@ -113,6 +113,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o | `ctx.hook_index` | Zero-based position in the authored hook list | | `ctx.run_id` | Current run ID | | `ctx.target` | Active target | +| `ctx.target_config` | The active target's database and schemas ([Target settings](/docs/concepts/python-nodes/overview#target-settings)) | | `ctx.vars` | Effective project variables | | `ctx.destination` | Destination relation metadata | | `ctx.relation(ref)` | Adapter-qualified relation for a [declared read](#declared-reads) | diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 02bb37969d..124ad222de 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -234,6 +234,7 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `current_cursor_value` | `object \| None` | Current `MAX(cursor_column)` from the destination, or `None` if the table does not exist or has no cursor column | | `run_id` | `str` | Unique identifier for this execution run | | `target` | `str \| None` | Active target name (e.g. `dev`, `prod`) | +| `target_config` | `TargetSettings` | The active target's database, schema and loader schema ([Target settings](/docs/concepts/python-nodes/overview#target-settings)) | | `vars` | `dict` | Project variables (merged from project, target, and local config) | | `is_reload` | `bool` | `True` when `--reload` was passed | | `start_cursor_ts` | `datetime \| None` | Timestamp cursor start override from `--start-cursor-ts` | diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 25068aab90..87c93836f3 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -97,6 +97,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset |--------|-------------| | `ctx.run_id` | Unique identifier for this run | | `ctx.target` | Active target name | +| `ctx.target_config` | The active target's database and schemas, as SQLBuild resolves them (see [Target settings](#target-settings)) | | `ctx.vars` | Project variables | | `ctx.is_reload` | `True` when `--reload` was passed | | `ctx.adapter` / `ctx.connection` | Adapter and live connection | @@ -112,6 +113,30 @@ Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check context Providers can also be injected directly as function parameters by name. See [Providers](/docs/concepts/python-nodes/providers) for details. +### Target settings + +Read the active target's locations from `ctx.target_config` instead of keeping each target's schema names in Python. The values are resolved the way SQLBuild resolves them for models and loaders, including `sqlbuild_local.toml` overrides, and are read-only: + +| Field | Value | +|-------|-------| +| `name` | Active target name, or `None` without targets | +| `database` | Database models build into, or `None` when the adapter has none or the value is `preserve` | +| `schema` | Schema models build into, or `None` when it is `preserve` | +| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema`; `None` when that schema uses a `${CTX:...}` template | +| `database_preserved` / `schema_preserved` | `True` when the target sets that value to `preserve` | + +```python +@asset +def partner_export(ctx): + """Export orders to the partner staging table.""" + target = ctx.target_config + staging = f"{target.database}.{target.loader_schema}.partner_staging" + ctx.execute_sql(f"CREATE OR REPLACE TABLE {staging} AS SELECT * FROM orders") + return ctx.result() +``` + +Connection settings and secrets are not part of `ctx.target_config`. + ## Returns and skips Tasks and assets return through `ctx.result(...)`: diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index b9b6efd791..3f6481302b 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -153,6 +153,23 @@ class WarehouseClient(Provider): Both `setup` and `teardown` are optional. A provider with only field declarations and no lifecycle methods is valid; it acts as a validated configuration object. +### The setup context + +In `sqb build`, `sqb check` and `sqb load`, `setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](/docs/concepts/python-nodes/overview#target-settings)): + +```python +class PartnerStage(Provider): + """Stage for partner exports.""" + + def setup(self, ctx): + target = ctx.target_config + self.stage = f"{target.database}.{target.loader_schema}.partner_stage" +``` + +Details: + +- A provider first used by an event exporter or command output sink, or by an orchestration sensor or calendar, is set up with `ctx` set to `None`. Check for `None` when such a provider also reads the target. + ## Configuration from environment variables Because `Provider` extends `pydantic-settings BaseSettings`, fields without defaults are read from environment variables. The environment variable name matches the field name in uppercase: