diff --git a/Cargo.lock b/Cargo.lock index 4726da4380..4c781910fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4604,6 +4604,7 @@ dependencies = [ "chrono", "chrono-tz", "pyo3", + "serde", "serde_json", "sha2 0.10.9", "sqlbuild-analysis", @@ -4623,6 +4624,7 @@ dependencies = [ "sqlbuild-python-front", "sqlbuild-rules", "sqlbuild-sqltext", + "sqlbuild-state", "tokio", "uuid", ] diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index afbf2f8d88..30781736f8 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -3,7 +3,7 @@ use sqlbuild_core::constants::STATE_TABLE_NAMES; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 12] = STATE_TABLE_NAMES; +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 13] = STATE_TABLE_NAMES; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; /// Separator between archive name segments. diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs index 62d35f1ef5..499c63845c 100644 --- a/crates/sqlbuild-cli/src/person_login/main/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod open_browser; pub(crate) mod read_node_sql; pub(crate) mod request_warehouse_sync; pub(crate) mod run_login; +pub mod signed_in_principal; pub mod sync_after_build; diff --git a/crates/sqlbuild-cli/src/person_login/main/signed_in_principal.rs b/crates/sqlbuild-cli/src/person_login/main/signed_in_principal.rs new file mode 100644 index 0000000000..1464d36ff2 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/signed_in_principal.rs @@ -0,0 +1,22 @@ +//! The principal a person is signed in as with `sqb login`, read from local credentials only. + +use std::path::Path; + +use chrono::Utc; +use sqlbuild_orchestration::api::main::parse_origin::parse_origin; + +use crate::person_login::_helpers::build_sync::configured_location; +use crate::person_login::_helpers::credentials; + +/// `principal:` of an unexpired session for the configured server; None when not signed in. +pub fn signed_in_principal(project_dir: &Path) -> Result, std::io::Error> { + let Some((server, _)) = configured_location(project_dir)? else { + return Ok(None); + }; + let origin: String = parse_origin(&server, "CLI").map_err(std::io::Error::other)?; + let Some(credential) = credentials::load(&origin)? else { + return Ok(None); + }; + Ok((credential.identity.expires_at > Utc::now()) + .then(|| format!("principal:{}", credential.identity.principal_id))) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 1c76005f82..7ceafd616b 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -15,7 +15,7 @@ use crate::project::_helpers::values::{int_below, python_equal}; use crate::project::constants::{ ALLOW_MISSING_SECRETS_KEY, ALLOW_WAREHOUSE_ROUND_TRIPS_KEY, DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, - IGNORE_AUDIT_FAILURES_KEY, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, + IGNORE_AUDIT_FAILURES_KEY, RECORD_RUN_HOST_KEY, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, }; use crate::project::constants::{ @@ -304,6 +304,14 @@ pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) -> false, )?)), ), + ( + RECORD_RUN_HOST_KEY, + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + RECORD_RUN_HOST_KEY, + true, + )?)), + ), ]; let mut fields = fields; for (key, seconds_key, default) in [ diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index ad8150d254..a6ad43ab52 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -55,8 +55,10 @@ pub(crate) const ALLOW_MISSING_SECRETS_KEY: &str = "allow_missing_secrets"; pub(crate) const SCENARIO_TIMEOUT_KEY: &str = "scenario_timeout"; /// Whether a failing audit holds nothing back; `[settings]` and each target may set it. pub(crate) const IGNORE_AUDIT_FAILURES_KEY: &str = "ignore_audit_failures"; +/// Whether each run records the machine it ran on in the runs state table. +pub(crate) const RECORD_RUN_HOST_KEY: &str = "record_run_host"; pub(crate) const DEFAULT_TEST_TIMEOUT: &str = "30s"; -pub(crate) const SETTINGS_KEYS: [&str; 18] = [ +pub(crate) const SETTINGS_KEYS: [&str; 19] = [ "sql_analysis", "require_sql_analysis", "query_change_tracking", @@ -75,6 +77,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 18] = [ "allow_missing_secrets", "allow_warehouse_round_trips", IGNORE_AUDIT_FAILURES_KEY, + RECORD_RUN_HOST_KEY, ]; pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "append_cursor_inclusive", diff --git a/crates/sqlbuild-core/src/build_command/constants.rs b/crates/sqlbuild-core/src/build_command/constants.rs index a7f88bc604..815bd9c52b 100644 --- a/crates/sqlbuild-core/src/build_command/constants.rs +++ b/crates/sqlbuild-core/src/build_command/constants.rs @@ -47,9 +47,12 @@ pub(crate) const C0_CONTROL_CODE_LIMIT: u32 = 0x20; pub(crate) const C1_CONTROL_CODE_START: u32 = 0x7f; pub(crate) const C1_CONTROL_CODE_LIMIT: u32 = 0xa0; -pub(crate) const PYTHON_NODE_SUCCESS: &str = "success"; -pub(crate) const PYTHON_NODE_SKIPPED: &str = "skipped"; -pub(crate) const PYTHON_NODE_FAILED: &str = "failed"; +/// A Python node or check that succeeded, warnings included. +pub const PYTHON_NODE_SUCCESS: &str = "success"; +/// A Python node or check that never ran. +pub const PYTHON_NODE_SKIPPED: &str = "skipped"; +/// A Python node or check that failed, which fails the build. +pub const PYTHON_NODE_FAILED: &str = "failed"; pub(crate) const PYTHON_STATUS_OK: &str = "OK"; pub(crate) const PYTHON_STATUS_SKIP: &str = "SKIP"; pub(crate) const PYTHON_STATUS_FAIL: &str = "FAIL"; diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 63fc1820a8..95966a7de9 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -42,8 +42,10 @@ pub const JANITOR_EVENTS_TABLE_NAME: &str = "_sqlbuild_janitor_events"; pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; /// State table of estimated query cost per run, resource and warehouse; display only. pub const RESOURCE_COSTS_TABLE_NAME: &str = "_sqlbuild_resource_costs"; +/// State table of each build's start and end, which shows runs the control plane did not start. +pub const RUNS_TABLE_NAME: &str = "_sqlbuild_runs"; /// Every state table SQLBuild writes in a target's state schema. -pub const STATE_TABLE_NAMES: [&str; 12] = [ +pub const STATE_TABLE_NAMES: [&str; 13] = [ FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, @@ -56,6 +58,7 @@ pub const STATE_TABLE_NAMES: [&str; 12] = [ OLD_NAME_VIEW_TABLE_NAME, OBSERVATIONS_TABLE_NAME, RESOURCE_COSTS_TABLE_NAME, + RUNS_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 3aea4d97c4..6d0b5e4ff7 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -3,7 +3,7 @@ use chrono::{DateTime, Duration, Utc}; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_core::constants::STATE_TABLE_NAMES; +use sqlbuild_core::constants::{RUNS_TABLE_NAME, STATE_TABLE_NAMES}; use uuid::Uuid; use crate::api::tests::changes_helpers::{connect, expect_topics}; @@ -36,7 +36,11 @@ use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; #[test] fn given_state_tables_when_classified_then_each_is_mirrored_by_exactly_one_kind() { - let mut written: Vec<&str> = STATE_TABLE_NAMES.to_vec(); + // The runs table has no mirror until the control store decides where observed runs live. + let mut written: Vec<&str> = STATE_TABLE_NAMES + .into_iter() + .filter(|table| *table != RUNS_TABLE_NAME) + .collect(); written.sort_unstable(); let test_cases = [StateTableClassificationTestCase { description: "every state table SQLBuild writes, with statistics rewinding to catalog", diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index c041a0efc2..86db23df0b 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -237,6 +237,7 @@ const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("allow_missing_secrets", FieldDefault::False), FieldSpec::new("allow_warehouse_round_trips", FieldDefault::False), FieldSpec::new("ignore_audit_failures", FieldDefault::False), + FieldSpec::new("record_run_host", FieldDefault::True), ], }; const R_SCOPES_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-project/Cargo.toml b/crates/sqlbuild-python-project/Cargo.toml index dc526705d0..c207f1cbe2 100644 --- a/crates/sqlbuild-python-project/Cargo.toml +++ b/crates/sqlbuild-python-project/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true chrono.workspace = true chrono-tz.workspace = true pyo3.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-analysis.workspace = true @@ -30,6 +31,7 @@ sqlbuild-python-base.workspace = true sqlbuild-python-front.workspace = true sqlbuild-rules.workspace = true sqlbuild-sqltext.workspace = true +sqlbuild-state.workspace = true tokio.workspace = true uuid.workspace = true diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 53d25310cf..50168e66a7 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -54,7 +54,15 @@ pub(crate) fn selected_jobs( selectors: &[String], catalog: &BTreeMap, ) -> BTreeSet { - let mut jobs: BTreeSet = BTreeSet::new(); + ordered_jobs(selectors, catalog).into_iter().collect() +} + +/// The jobs `selectors` name, each once in the order first named. +pub(crate) fn ordered_jobs( + selectors: &[String], + catalog: &BTreeMap, +) -> Vec { + let mut jobs: Vec = Vec::new(); let parts = selectors .iter() .flat_map(|selector| selector.split(char::is_whitespace)) @@ -63,8 +71,11 @@ pub(crate) fn selected_jobs( let Ok(ParsedSelector::Kind { kind, value, .. }) = parse_selector(part) else { continue; }; - if JOB_SELECTOR_KINDS.contains(&kind.as_str()) && catalog.contains_key(&value) { - jobs.insert(value); + if JOB_SELECTOR_KINDS.contains(&kind.as_str()) + && catalog.contains_key(&value) + && !jobs.contains(&value) + { + jobs.push(value); } } jobs diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index cbc2c7c0c7..0cb998a7ee 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -7,6 +7,7 @@ pub mod install; pub mod job_names; pub mod materialization; pub mod model_facts; +pub mod named_jobs; pub mod needed_jobs; pub mod plain_value; pub mod python_column_fact; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs new file mode 100644 index 0000000000..b79ebfbee1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs @@ -0,0 +1,12 @@ +//! The jobs a command's selectors name, in the order named. + +use crate::compiled_project::_helpers::job_catalog::job_catalog; +use crate::compiled_project::_helpers::job_catalog::ordered_jobs; +use pyo3::prelude::{PyResult, Python}; + +/// The jobs `selectors` name, as `job:` or a bare job name, each once in the order named; none outside a command. +pub fn named_jobs(py: Python<'_>, selectors: &[String]) -> PyResult> { + Ok(job_catalog(py)? + .map(|(catalog, _)| ordered_jobs(selectors, &catalog)) + .unwrap_or_default()) +} diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs index b336f64d4d..960edd15e0 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod orchestration_triggers; +pub(crate) mod runs_state; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs new file mode 100644 index 0000000000..effdc06fcc --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs @@ -0,0 +1,194 @@ +//! Helpers of the runs table recorder: its DDL, the run's identity facts and its row cells. + +use crate::compiled_project::main::named_jobs::named_jobs; +use crate::native_host::models::RunCommandScope; +use crate::native_host::models::RunIdentity; +use pyo3::exceptions::{PyKeyError, PyOSError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_cli::person_login::main::signed_in_principal::signed_in_principal; +use sqlbuild_core::constants::RUNS_TABLE_NAME; +use sqlbuild_core::git_revisions::main::head_commit::head_commit; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::main::write_console::write_console; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::main::fixed_state_table_sql::fixed_state_table_sql; +use sqlbuild_python_front::native_host::main::integer_literal::integer_literal; +use sqlbuild_python_front::native_host::main::literal::literal; +use sqlbuild_state::runs::constants::{FIRST_ATTEMPT, NULLABLE_RUN_COLUMNS, RUN_COLUMN_TYPES}; +use sqlbuild_state::runs::models::{RunNodeCounts, ServerRun}; +use std::path::Path; +use std::process::{Command, Stdio}; + +/// The adapter method that marks the runs table as supported. +pub(crate) const CREATE_TABLE_METHOD: &str = "render_create_runs_table_sql"; +/// The adapter method rendering the session's warehouse identity as SQL, or None without one. +pub(crate) const WAREHOUSE_USER_METHOD: &str = "render_current_warehouse_user"; + +/// Create-if-missing DDL for the runs table, with adapter-rendered column types. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +pub(crate) fn build_runs_create_table_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + render_framework_type: &Bound<'_, PyAny>, +) -> PyResult { + fixed_state_table_sql( + (database, schema), + (render_qualified_name, render_framework_type), + (RUNS_TABLE_NAME, &RUN_COLUMN_TYPES, &NULLABLE_RUN_COLUMNS), + ) +} + +/// Whether the checkout at `directory` has changes; None when Git is not installed or cannot tell. +pub(crate) fn git_dirty(directory: &Path) -> std::io::Result> { + let output = match Command::new("git") + .current_dir(directory) + .args(["status", "--porcelain", "--untracked-files=normal"]) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + { + Ok(output) => output, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + Ok(output + .status + .success() + .then(|| !String::from_utf8_lossy(&output.stdout).trim().is_empty())) +} + +/// The strings of a Python iterable of selectors. +pub(crate) fn texts(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +/// The keys of `--vars` in the order given; their values can hold secrets and are never read. +pub(crate) fn var_keys(cli_vars: &Bound<'_, PyAny>) -> PyResult> { + if cli_vars.is_none() { + return Ok(Vec::new()); + } + cli_vars + .downcast::() + .map_err(|_| PyValueError::new_err("--vars is not a mapping"))? + .iter() + .map(|(key, _)| key.str()?.extract()) + .collect() +} + +/// The text of a zero-argument Python call; None when the system cannot say (`OSError` or `KeyError`). +pub(crate) fn python_fact( + py: Python<'_>, + (module, name): (&str, &str), +) -> PyResult> { + match imported(py, module, name)?.call0() { + Ok(value) => Ok(Some(value.extract::()?).filter(|value| !value.is_empty())), + Err(error) + if error.is_instance_of::(py) || error.is_instance_of::(py) => + { + Ok(None) + } + Err(error) => Err(error), + } +} + +/// The `sqb login` principal; a credential store that cannot be read only warns. +pub(crate) fn principal(py: Python<'_>, project_dir: &Path) -> Option { + match signed_in_principal(project_dir) { + Ok(principal) => principal, + Err(error) => { + let _ = write_console( + py, + "stderr", + &format!("Warning: this run records no sqlbuild_user: {error}\n"), + ); + None + } + } +} + +/// The identity of one run from its request; `server` is set when a worker runs it for the server. +pub(crate) fn run_identity( + request: &Bound<'_, PyAny>, + scope: &RunCommandScope<'_>, + (run_id, target_name): (String, Option), + server: Option, +) -> PyResult { + let py: Python<'_> = request.py(); + let project_dir: &Path = scope.project_dir; + let select: Vec = texts(&native_field(request, "select")?)?; + let (git_commit, git_dirty) = py.detach(|| -> std::io::Result<_> { + let commit: Option = head_commit(project_dir); + let dirty: Option = match commit { + Some(_) => git_dirty(project_dir)?, + None => None, + }; + Ok((commit, dirty)) + })?; + let sqlbuild_user: Option = match &server { + Some(run) => Some(run.sqlbuild_user.clone()), + None => principal(py, project_dir), + }; + Ok(RunIdentity { + run_id, + attempt: server.as_ref().map_or(FIRST_ATTEMPT, |run| run.attempt), + retry_of_run_id: server.as_ref().and_then(|run| run.retry_of_run_id.clone()), + orchestration_run_id: server.map(|run| run.orchestration_run_id), + command: scope.command, + sqlbuild_version: scope.sqlbuild_version, + named_jobs: named_jobs(py, &select)?, + select, + exclude: texts(&native_field(request, "exclude")?)?, + target_name, + sqlbuild_user, + os_user: python_fact(py, ("getpass", "getuser"))?, + host: if scope.record_host { + python_fact(py, ("socket", "gethostname"))? + } else { + None + }, + git_commit, + git_dirty, + var_keys: var_keys(&native_field(request, "cli_vars")?)?, + }) +} + +pub(crate) fn json_text(value: &T) -> PyResult { + serde_json::to_string(value).map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// One row's cells in `RUN_COLUMN_TYPES` order; `finish` fills the finished row's status and counts. +pub(crate) fn row_cells( + py: Python<'_>, + identity: &RunIdentity, + (phase, finish): (&str, Option<(&str, &RunNodeCounts)>), + (warehouse_user, recorded_at): (&str, &str), +) -> PyResult> { + let counts: Option = finish.map(|(_, counts)| json_text(counts)).transpose()?; + Ok(vec![ + literal(py, Some(&identity.run_id))?, + literal(py, Some(phase))?, + integer_literal(py, Some(identity.attempt))?, + literal(py, identity.retry_of_run_id.as_deref())?, + literal(py, identity.orchestration_run_id.as_deref())?, + literal(py, Some(identity.command))?, + literal(py, Some(&json_text(&identity.select)?))?, + literal(py, Some(&json_text(&identity.exclude)?))?, + literal(py, Some(&json_text(&identity.named_jobs)?))?, + literal(py, identity.target_name.as_deref())?, + warehouse_user.to_owned(), + literal(py, identity.sqlbuild_user.as_deref())?, + literal(py, identity.os_user.as_deref())?, + literal(py, identity.host.as_deref())?, + literal(py, Some(identity.sqlbuild_version))?, + literal(py, identity.git_commit.as_deref())?, + integer_literal(py, identity.git_dirty.map(i64::from))?, + literal(py, Some(&json_text(&identity.var_keys)?))?, + literal(py, finish.map(|(status, _)| status))?, + literal(py, counts.as_deref())?, + recorded_at.to_owned(), + ]) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs index 9d80042f90..2da4595b3e 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -1,2 +1,4 @@ pub mod compile_calendars; pub mod compile_sensors; +pub mod register_runs_state; +pub mod run_recorder; diff --git a/crates/sqlbuild-python-project/src/native_host/main/register_runs_state.rs b/crates/sqlbuild-python-project/src/native_host/main/register_runs_state.rs new file mode 100644 index 0000000000..7a6fe6b51f --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/register_runs_state.rs @@ -0,0 +1,10 @@ +//! Register the runs table functions. + +use crate::native_host::_helpers::runs_state::build_runs_create_table_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the runs table functions. +pub fn register_runs_state(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_runs_create_table_sql, module)?) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs b/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs new file mode 100644 index 0000000000..a4959440f4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs @@ -0,0 +1,72 @@ +//! One run's recorder of the runs table, the same for the CLI and a worker. + +use crate::native_host::_helpers::runs_state::run_identity; +use crate::native_host::models::{RunCommandScope, RunIdentity, RunRecorder}; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use sqlbuild_core::constants::RUNS_TABLE_NAME; +use sqlbuild_python_base::native_host::main::write_console::write_console; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::runs::models::ServerRun; +use std::path::PathBuf; + +/// A run's recorder with its started row written, or none for a `preserve` schema; only an interrupt errors. +pub fn run_recorder<'py>( + (command, sqlbuild_version): (&'static str, &'static str), + (request, invocation, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + server: Option, +) -> PyResult>> { + let location = |value: Option| value.filter(|value| value != PRESERVE_TARGET_VALUE); + let opened = || -> PyResult>> { + let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; + let settings: Bound<'py, PyAny> = native_field( + &native_field( + &native_field(invocation, "discovered_inputs")?, + "project_config", + )?, + "settings", + )?; + let scope = RunCommandScope { + command, + project_dir: &project_dir, + record_host: native_field(&settings, "record_run_host")?.is_truthy()?, + sqlbuild_version, + }; + let identity: RunIdentity = run_identity( + request, + &scope, + ( + native_field(project, "run_id")?.str()?.extract()?, + native_field(project, "effective_target_name")?.extract()?, + ), + server, + )?; + RunRecorder::open( + ( + &native_field(invocation, "adapter")?, + &native_field(invocation, "connection_config")?, + ), + ( + location(native_field(project, "effective_target_database")?.extract()?), + location(native_field(project, "effective_target_schema")?.extract()?), + ), + identity, + ) + }; + match opened() { + Err(error) if error.is_instance_of::(request.py()) => { + let _ = write_console( + request.py(), + "stderr", + &format!("Warning: this run is not recorded in {RUNS_TABLE_NAME}: {error}\n"), + ); + Ok(None) + } + Ok(Some(recorder)) => { + recorder.started()?; + Ok(Some(recorder)) + } + opened => opened, + } +} diff --git a/crates/sqlbuild-python-project/src/native_host/models.rs b/crates/sqlbuild-python-project/src/native_host/models.rs index 70a2b87d14..c982fe1c73 100644 --- a/crates/sqlbuild-python-project/src/native_host/models.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -1,15 +1,34 @@ //! Public records of native host. use crate::compiled_views::models::NativeCompiledModel; +use crate::native_host::_helpers::runs_state::CREATE_TABLE_METHOD; +use crate::native_host::_helpers::runs_state::WAREHOUSE_USER_METHOD; +use crate::native_host::_helpers::runs_state::row_cells; use crate::planner_scope::models::ScopeData; use pyo3::IntoPyObject; +use pyo3::PyErr; +use pyo3::exceptions::{PyException, PyKeyboardInterrupt}; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; use pyo3::prelude::PyResult; use pyo3::prelude::Python; +use sqlbuild_core::constants::RUNS_TABLE_NAME; use sqlbuild_orchestration::declarations::models::TriggerDefinition; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::write_console::write_console; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_front::native_host::main::execute::execute; +use sqlbuild_python_front::native_host::main::literal::literal; +use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_state::runs::constants::{FINISHED_PHASE, RUN_COLUMN_TYPES, STARTED_PHASE}; +use sqlbuild_state::runs::main::run_status::run_status; +use sqlbuild_state::runs::models::{RunEnding, RunNodeCounts}; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use std::path::Path; use std::sync::Arc; /// Native compiled views stay typed; caller-authored protocol implementations remain public values. @@ -124,3 +143,196 @@ impl DeferredCost { .take()) } } + +/// What a run records about itself in both of its rows; `--vars` contributes only its keys. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunIdentity { + pub run_id: String, + pub attempt: i64, + pub retry_of_run_id: Option, + pub orchestration_run_id: Option, + pub command: &'static str, + /// The SQLBuild version that recorded the run. + pub sqlbuild_version: &'static str, + pub select: Vec, + pub exclude: Vec, + pub named_jobs: Vec, + pub target_name: Option, + pub sqlbuild_user: Option, + pub os_user: Option, + pub host: Option, + pub git_commit: Option, + pub git_dirty: Option, + pub var_keys: Vec, +} + +/// One command's run: its name, project directory and whether the project records the host. +#[derive(Debug)] +pub struct RunCommandScope<'a> { + pub command: &'static str, + pub project_dir: &'a Path, + pub record_host: bool, + pub sqlbuild_version: &'static str, +} + +/// A build's two rows in its target's runs table; a failed write warns and never fails the build. +#[derive(Debug)] +pub struct RunRecorder<'py> { + adapter: Bound<'py, PyAny>, + connection_config: Bound<'py, PyAny>, + location: Location, + table: String, + identity: RunIdentity, +} + +impl<'py> RunRecorder<'py> { + /// A recorder when the adapter keeps a runs table and the target has a schema; never connects. + pub fn open( + (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (Option, Option), + identity: RunIdentity, + ) -> PyResult> { + let Some(schema) = schema else { + return Ok(None); + }; + if !adapter.hasattr(CREATE_TABLE_METHOD)? { + return Ok(None); + } + let location = Location { database, schema }; + let keywords = location.keywords(adapter.py()); + let Some(table) = state_table_name( + &native_field(adapter, "render_qualified_name")?, + (&keywords[0].1, &keywords[1].1), + RUNS_TABLE_NAME, + )? + else { + return Ok(None); + }; + Ok(Some(Self { + adapter: adapter.clone(), + connection_config: connection_config.clone(), + location, + table, + identity, + })) + } + + /// Set the table up once for the run and record that it started; only an interrupt escapes. + pub fn started(&self) -> PyResult<()> { + self.warn_on_error(STARTED_PHASE, self.write(STARTED_PHASE, None, true)) + } + + /// Record how the run ended and what its nodes did; only an interrupt escapes as an error. + pub fn finished(&self, counts: &RunNodeCounts, ending: RunEnding) -> PyResult<()> { + let finish = (run_status(counts, ending), counts); + self.warn_on_error( + FINISHED_PHASE, + self.append_row(FINISHED_PHASE, Some(finish)), + ) + } + + /// Record a run that stopped on `error`: cancelled on an interrupt, else failed. Never errors. + pub fn raised(&self, error: &PyErr) { + let ending: RunEnding = if error.is_instance_of::(self.adapter.py()) { + RunEnding::Interrupted + } else { + RunEnding::Raised + }; + let _ = self.finished(&RunNodeCounts::default(), ending); + } + + /// Insert directly, as the started row set the table up, and set it up again only when that fails. + fn append_row(&self, phase: &str, finish: Option<(&str, &RunNodeCounts)>) -> PyResult<()> { + self.write(phase, finish, false).or_else(|error| { + if error.is_instance_of::(self.adapter.py()) { + self.write(phase, finish, true) + } else { + Err(error) + } + }) + } + + /// Warn on a failed write and carry on; an interrupt is not a failed write, so it propagates. + fn warn_on_error(&self, phase: &str, written: PyResult<()>) -> PyResult<()> { + let py: Python<'py> = self.adapter.py(); + match written { + Err(error) if error.is_instance_of::(py) => { + let _ = write_console( + py, + "stderr", + &format!( + "Warning: could not record that run {} {phase} in {RUNS_TABLE_NAME}: {error}\n", + self.identity.run_id + ), + ); + Ok(()) + } + written => written, + } + } + + /// Connect for this row only, so loaders can still lock file warehouses during the build. + fn write( + &self, + phase: &str, + finish: Option<(&str, &RunNodeCounts)>, + ensure_table: bool, + ) -> PyResult<()> { + let connection: Bound<'py, PyAny> = + native_field(&self.adapter, "connect")?.call1((&self.connection_config,))?; + let written: PyResult<()> = self.append(&connection, (phase, finish), ensure_table); + let closed: PyResult> = + native_field(&self.adapter, "close")?.call1((&connection,)); + written?; + closed.map(|_| ()) + } + + fn append( + &self, + connection: &Bound<'py, PyAny>, + row: (&str, Option<(&str, &RunNodeCounts)>), + ensure_table: bool, + ) -> PyResult<()> { + let py: Python<'py> = self.adapter.py(); + if ensure_table { + let keywords = self.location.keywords(py); + for statement in call_with( + &native_field(&self.adapter, "render_create_schema")?, + &keywords, + )? + .try_iter()? + { + execute(&self.adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(&self.adapter, CREATE_TABLE_METHOD)?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(&self.adapter, connection, &create)?; + } + } + let recorded_at: String = call_with( + &native_field(&self.adapter, "render_current_utc_timestamp")?, + &[], + )? + .extract()?; + let warehouse_user: String = + match call_with(&native_field(&self.adapter, WAREHOUSE_USER_METHOD)?, &[])? + .extract::>()? + { + Some(expression) => expression, + None => literal(py, None)?, + }; + let columns: Vec<&str> = RUN_COLUMN_TYPES.iter().map(|(column, _)| *column).collect(); + let cells: Vec = + row_cells(py, &self.identity, row, (&warehouse_user, &recorded_at))?; + execute( + &self.adapter, + connection, + &values_insert_sql(&self.table, &columns, &[cells]), + ) + .map(|_| ()) + } +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/models.rs b/crates/sqlbuild-python-project/src/project_snapshot/models.rs index 385f0b8903..2070d4e7df 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/models.rs @@ -35,6 +35,7 @@ use sqlbuild_orchestration::runs::models::NodeOutcome; use sqlbuild_orchestration::runs::models::RunChange; use sqlbuild_orchestration::runs::models::RunCommand; use sqlbuild_orchestration::runs::models::RunExecution; +use sqlbuild_orchestration::runs::models::RunOrigin; use sqlbuild_orchestration::runs::models::RunOutcome; use sqlbuild_orchestration::snapshots::main::publication::publish_remote_snapshot::publish_remote_snapshot; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -43,6 +44,11 @@ use sqlbuild_orchestration::worker::errors::WorkerError; use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_state::runs::constants::FIRST_ATTEMPT; +use sqlbuild_state::runs::constants::SYSTEM_RETRY_USER; +use sqlbuild_state::runs::constants::SYSTEM_SCHEDULE_USER; +use sqlbuild_state::runs::constants::SYSTEM_SENSOR_USER; +use sqlbuild_state::runs::models::ServerRun; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::VecDeque; @@ -184,6 +190,29 @@ impl WorkerBuild { self.execution.claim.request.run_id } + /// What the runs table records of the server run this build executes. + pub fn server_run(&self) -> ServerRun { + let request = &self.execution.claim.request; + let retry_of_run_id: Option = request + .retry + .and(request.rerun_of_run_id) + .map(|run_id| run_id.to_string()); + let sqlbuild_user: String = match (request.origin, request.retry) { + (_, Some(_)) => SYSTEM_RETRY_USER.to_owned(), + (RunOrigin::Schedule, None) => SYSTEM_SCHEDULE_USER.to_owned(), + (RunOrigin::Sensor, None) => SYSTEM_SENSOR_USER.to_owned(), + _ => request.requested_by.clone(), + }; + ServerRun { + orchestration_run_id: request.run_id.to_string(), + sqlbuild_user, + attempt: request + .retry + .map_or(FIRST_ATTEMPT, |retry| i64::from(retry.attempt) + 1), + retry_of_run_id, + } + } + /// Keep the build's cost finalization to collect after the run's outcome is published. pub fn defer_cost(&self, finalization: &Bound<'_, PyAny>) -> PyResult<()> { self.deferred_cost.defer(finalization) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 866799cfe4..4ac4bf03ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -7,9 +7,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cli::person_login::main::sync_after_build::sync_after_build; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; +use sqlbuild_state::runs::constants::BUILD_COMMAND; use crate::bindings::_helpers::build_command::completion::{ - complete_direct_build, finalize_exceptional_cost, finalize_no_work_with_timings, + complete_direct_build, finalize_exceptional_cost, finalize_no_work_with_timings, finished_run, record_pending_cost, write_partial_timings, }; use crate::bindings::_helpers::build_command::execution::execute_direct_build; @@ -37,6 +38,8 @@ use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_planner::observations::main::arm_source_capture::arm_source_capture; use sqlbuild_python_planner::observations::main::take_source_capture::take_source_capture; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::native_host::main::run_recorder::run_recorder; +use sqlbuild_python_project::native_host::models::RunRecorder; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { @@ -164,14 +167,32 @@ fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) return Ok(0); } let started_at: Bound<'_, PyAny> = record_pending_cost(context, &pipeline)?; - match execute_direct_build(context, &pipeline, providers) { - Ok((preparation, outcome, checks)) => complete_direct_build( - context, - &pipeline, - (&preparation, &outcome, &checks), - &started_at, + let recorder: Option> = run_recorder( + (BUILD_COMMAND, env!("CARGO_PKG_VERSION")), + ( + &context.request, + &context.invocation, + &native_field(&pipeline, "project")?, ), + context.worker.as_ref().map(|worker| worker.server_run()), + )?; + match execute_direct_build(context, &pipeline, providers) { + Ok((preparation, outcome, checks)) => { + if let Some(recorder) = &recorder { + let (counts, ending) = finished_run(&outcome, &checks)?; + recorder.finished(&counts, ending)?; + } + complete_direct_build( + context, + &pipeline, + (&preparation, &outcome, &checks), + &started_at, + ) + } Err(error) => { + if let Some(recorder) = &recorder { + recorder.raised(&error); + } finalize_exceptional_cost(context, &pipeline, (&started_at, &error)); Err(error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index c7e486bdbf..e1eaf4b63f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -11,6 +11,7 @@ use sqlbuild_core::build_command::models::PhaseTimings; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::BuildStatus; +use sqlbuild_state::runs::models::{RunEnding, RunNodeCounts}; use crate::bindings::_helpers::build_command::execution::Outcome; use crate::bindings::_helpers::build_command::execution::Preparation; @@ -418,6 +419,36 @@ fn check_node_text(check: &CheckResultText) -> PythonNodeText { } } +/// A completed build's footer node counts, and `Raised` when it failed without a failed node. +pub(crate) fn finished_run( + outcome: &Outcome<'_>, + check_results: &Bound<'_, PyAny>, +) -> PyResult<(RunNodeCounts, RunEnding)> { + let result = &outcome.result; + let mut counts: RunNodeCounts = RunNodeCounts { + succeeded: result.success_count, + warned: result.warning_count, + failed: result.failure_count, + skipped: result.skipped_count, + not_started: 0, + }; + for node in outcome.python_results.try_iter()? { + let node: Bound<'_, PyAny> = node?; + let status: String = native_field(&node, "status")?.str()?.extract()?; + counts = counts.with_python_node(&status, native_field(&node, "warning_messages")?.len()?); + } + for check in check_results.try_iter()? { + let check: PythonNodeText = check_node_text(&check_result_text(&check?)?); + counts = counts.with_python_node(&check.status, check.warnings.len()); + } + let ending: RunEnding = if result.status != BuildStatus::Success && counts.failed == 0 { + RunEnding::Raised + } else { + RunEnding::Completed + }; + Ok((counts, ending)) +} + /// Persist and render a completed direct build and return its exit code. pub(crate) fn complete_direct_build<'py>( context: &BuildContext<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index c4983c66af..10b753b5a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -63,7 +63,7 @@ const PROJECT_CONFIG_ELSEWHERE: [&str; 11] = [ ]; const LOCAL_CONFIG_ELSEWHERE: [&str; 4] = ["settings", "setting_overrides", "vars", "scenario"]; /// Settings that tune how a run executes or validates, not what a node computes. -const RUNTIME_SETTINGS: [&str; 11] = [ +const RUNTIME_SETTINGS: [&str; 12] = [ "sql_analysis", "require_sql_analysis", "allow_warehouse_round_trips", @@ -75,6 +75,7 @@ const RUNTIME_SETTINGS: [&str; 11] = [ "scenario_timeout_seconds", "delta_compile", "ignore_audit_failures", + "record_run_host", ]; const PYTHON_SUFFIX: &str = ".py"; /// Discovered Python node collections, in `PYTHON_NODE_KINDS` order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 0db154308e..9962d9ccc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -33,6 +33,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { sqlbuild_python_front::native_host::main::register_observations_state::register_observations_state(module)?; sqlbuild_python_front::resource_costs::main::register_resource_costs_state::register_resource_costs_state(module)?; sqlbuild_python_front::native_host::main::register_audit_results_state::register_audit_results_state(module)?; + sqlbuild_python_project::native_host::main::register_runs_state::register_runs_state(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; execution_history::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 54a5ec5f20..f63a0244ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -21,9 +21,9 @@ use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, - PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, SETTINGS_SECTION, - SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, - ZERO_DAY_DURATION, + PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, RUNS_TABLE_NAME, + SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, + TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, @@ -94,6 +94,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("AUDIT_RESULTS_TABLE_NAME", AUDIT_RESULTS_TABLE_NAME), ("JANITOR_EVENTS_TABLE_NAME", JANITOR_EVENTS_TABLE_NAME), ("OBSERVATIONS_TABLE_NAME", OBSERVATIONS_TABLE_NAME), + ("RUNS_TABLE_NAME", RUNS_TABLE_NAME), ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 1f231727c1..7cbfff7a89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -16,6 +16,8 @@ use sqlbuild_core::seed_command::main::render_seed_row::render_seed_row; use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; use sqlbuild_core::seed_command::models::SeedRow; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_state::runs::constants::SEED_COMMAND; +use sqlbuild_state::runs::models::{RunEnding, RunNodeCounts}; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; @@ -39,6 +41,8 @@ use sqlbuild_python_output::result_output::main::results::write_execution_json_o use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_planner::planner_entries::main::plans::build_seed_command_plan::build_seed_command_plan; use sqlbuild_python_planner::planner_entries::main::plans::read_selected_seed_fingerprints::read_selected_seed_fingerprints; +use sqlbuild_python_project::native_host::main::run_recorder::run_recorder; +use sqlbuild_python_project::native_host::models::RunRecorder; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; @@ -479,17 +483,47 @@ pub(crate) fn run_seed<'py>( .limit .max(1); write_header(&invocation, &pipeline, concurrency)?; + let recorder: Option> = run_recorder( + (SEED_COMMAND, env!("CARGO_PKG_VERSION")), + (request, &invocation, &native_field(&pipeline, "project")?), + worker.as_ref().map(|worker| worker.server_run()), + )?; let started: Instant = Instant::now(); - let results: Vec> = load_seeds(&invocation, &pipeline, (concurrency, worker))?; + let loaded: PyResult>> = + load_seeds(&invocation, &pipeline, (concurrency, worker)); + let results: Vec> = match loaded { + Ok(results) => results, + Err(error) => { + if let Some(recorder) = &recorder { + recorder.raised(&error); + } + return Err(error); + } + }; let elapsed: f64 = started.elapsed().as_secs_f64(); let mut passed: usize = 0; let mut failed: usize = 0; + let mut warned: usize = 0; for result in &results { match status_value(result)?.as_str() { SUCCESS_STATUS => passed += 1, FAILED_STATUS => failed += 1, _ => {} } + warned += native_field(result, "warning_messages")?.len()?; + } + if let Some(recorder) = &recorder { + let count = |value: usize| i64::try_from(value).unwrap_or(i64::MAX); + recorder.finished( + &RunNodeCounts { + succeeded: count(passed), + warned: count(warned), + failed: count(failed), + skipped: count(results.len() - passed - failed), + not_started: 0, + }, + RunEnding::Completed, + )?; } let stream: Bound<'_, PyAny> = native_field(&invocation, "progress_stream")?; let style: CliStyle = CliStyle { diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 2f5c84c43d..d31035afec 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -10,5 +10,6 @@ pub mod freshness; pub mod microbatches; pub mod observations; pub mod resource_costs; +pub mod runs; pub mod state_tables; pub mod warehouse_state; diff --git a/crates/sqlbuild-state/src/runs/constants.rs b/crates/sqlbuild-state/src/runs/constants.rs new file mode 100644 index 0000000000..5b5246edb5 --- /dev/null +++ b/crates/sqlbuild-state/src/runs/constants.rs @@ -0,0 +1,69 @@ +//! Columns, phases and statuses of the append-only runs state table. + +use crate::consumption::constants::INTEGER_STATE_VALUE_TYPE; +use crate::freshness::constants::{STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE}; + +/// The column the warehouse clock fills on insert. +pub const RECORDED_AT_COLUMN: &str = "recorded_at"; +/// Columns in declaration order with their state value types; `recorded_at` is last. +pub const RUN_COLUMN_TYPES: [(&str, &str); 21] = [ + ("run_id", STRING_STATE_VALUE_TYPE), + ("phase", STRING_STATE_VALUE_TYPE), + ("attempt", INTEGER_STATE_VALUE_TYPE), + ("retry_of_run_id", STRING_STATE_VALUE_TYPE), + ("orchestration_run_id", STRING_STATE_VALUE_TYPE), + ("command", STRING_STATE_VALUE_TYPE), + ("select_json", STRING_STATE_VALUE_TYPE), + ("exclude_json", STRING_STATE_VALUE_TYPE), + ("named_jobs_json", STRING_STATE_VALUE_TYPE), + ("target_name", STRING_STATE_VALUE_TYPE), + ("warehouse_user", STRING_STATE_VALUE_TYPE), + ("sqlbuild_user", STRING_STATE_VALUE_TYPE), + ("os_user", STRING_STATE_VALUE_TYPE), + ("host", STRING_STATE_VALUE_TYPE), + ("sqlbuild_version", STRING_STATE_VALUE_TYPE), + ("git_commit", STRING_STATE_VALUE_TYPE), + ("git_dirty", INTEGER_STATE_VALUE_TYPE), + ("var_keys_json", STRING_STATE_VALUE_TYPE), + ("status", STRING_STATE_VALUE_TYPE), + ("node_counts_json", STRING_STATE_VALUE_TYPE), + (RECORDED_AT_COLUMN, TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; +/// Columns that may be NULL: links, identities and facts not known or recorded, and finished-only columns. +pub const NULLABLE_RUN_COLUMNS: [&str; 11] = [ + "retry_of_run_id", + "orchestration_run_id", + "target_name", + "warehouse_user", + "sqlbuild_user", + "os_user", + "host", + "git_commit", + "git_dirty", + "status", + "node_counts_json", +]; +/// Phase of the row written before the first node runs. +pub const STARTED_PHASE: &str = "started"; +/// Phase of the row written once the run ends, however it ends. +pub const FINISHED_PHASE: &str = "finished"; +/// A finished run whose nodes all succeeded, warnings included. +pub const SUCCEEDED_STATUS: &str = "succeeded"; +/// A finished run in which nothing succeeded and something failed or was skipped. +pub const FAILED_STATUS: &str = "failed"; +/// A finished run stopped by an interrupt or a cancel. +pub const CANCELLED_STATUS: &str = "cancelled"; +/// A finished run in which some nodes succeeded and others failed or were skipped. +pub const PARTIAL_STATUS: &str = "partial"; +/// `sqlbuild_user` of a run the scheduler started. +pub const SYSTEM_SCHEDULE_USER: &str = "system:schedule"; +/// `sqlbuild_user` of a run a sensor started. +pub const SYSTEM_SENSOR_USER: &str = "system:sensor"; +/// `sqlbuild_user` of an automatic retry of a failed run. +pub const SYSTEM_RETRY_USER: &str = "system:retry"; +/// The attempt number of a run that is not a retry. +pub const FIRST_ATTEMPT: i64 = 1; +/// `command` of a run of `sqb build`. +pub const BUILD_COMMAND: &str = "build"; +/// `command` of a run of `sqb seed`. +pub const SEED_COMMAND: &str = "seed"; diff --git a/crates/sqlbuild-state/src/runs/main/mod.rs b/crates/sqlbuild-state/src/runs/main/mod.rs new file mode 100644 index 0000000000..bf28b9cd60 --- /dev/null +++ b/crates/sqlbuild-state/src/runs/main/mod.rs @@ -0,0 +1 @@ +pub mod run_status; diff --git a/crates/sqlbuild-state/src/runs/main/run_status.rs b/crates/sqlbuild-state/src/runs/main/run_status.rs new file mode 100644 index 0000000000..92f2c9b2dc --- /dev/null +++ b/crates/sqlbuild-state/src/runs/main/run_status.rs @@ -0,0 +1,17 @@ +//! A finished run's status from how it ended and its node counts, by the server's rule for its own runs. + +use crate::runs::constants::{CANCELLED_STATUS, FAILED_STATUS, PARTIAL_STATUS, SUCCEEDED_STATUS}; +use crate::runs::models::{RunEnding, RunNodeCounts}; + +/// `cancelled` if interrupted, `failed` on an error, else partial, failed or succeeded by node counts. +#[must_use] +pub fn run_status(counts: &RunNodeCounts, ending: RunEnding) -> &'static str { + let good: bool = counts.succeeded > 0; + let bad: bool = counts.failed + counts.skipped + counts.not_started > 0; + match (ending, bad, good) { + (RunEnding::Interrupted, _, _) => CANCELLED_STATUS, + (RunEnding::Raised, _, _) | (RunEnding::Completed, true, false) => FAILED_STATUS, + (RunEnding::Completed, true, true) => PARTIAL_STATUS, + (RunEnding::Completed, false, _) => SUCCEEDED_STATUS, + } +} diff --git a/crates/sqlbuild-state/src/runs/mod.rs b/crates/sqlbuild-state/src/runs/mod.rs new file mode 100644 index 0000000000..042268f15e --- /dev/null +++ b/crates/sqlbuild-state/src/runs/mod.rs @@ -0,0 +1,6 @@ +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/runs/models.rs b/crates/sqlbuild-state/src/runs/models.rs new file mode 100644 index 0000000000..740cd08d87 --- /dev/null +++ b/crates/sqlbuild-state/src/runs/models.rs @@ -0,0 +1,56 @@ +//! The node counts a finished run records. + +use serde::Serialize; +use sqlbuild_core::build_command::constants::{ + PYTHON_NODE_FAILED, PYTHON_NODE_SKIPPED, PYTHON_NODE_SUCCESS, +}; + +/// How many of a run's nodes ended each way; `not_started` counts planned nodes that never ran. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RunNodeCounts { + pub succeeded: i64, + pub warned: i64, + pub failed: i64, + pub skipped: i64, + pub not_started: i64, +} + +impl RunNodeCounts { + /// These counts with one more Python node or check of `status`, which warned `warnings` times. + #[must_use] + pub fn with_python_node(self, status: &str, warnings: usize) -> Self { + match status { + PYTHON_NODE_FAILED => Self { + failed: self.failed + 1, + ..self + }, + PYTHON_NODE_SKIPPED => Self { + skipped: self.skipped + 1, + ..self + }, + PYTHON_NODE_SUCCESS => Self { + succeeded: self.succeeded + 1, + warned: self.warned + i64::try_from(warnings).unwrap_or(i64::MAX), + ..self + }, + _ => self, + } + } +} + +/// What the server knows of a run it started; a worker's handle supplies it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ServerRun { + pub orchestration_run_id: String, + pub sqlbuild_user: String, + pub attempt: i64, + pub retry_of_run_id: Option, +} + +/// How a run ended: it completed, was interrupted or cancelled, or stopped on an error. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RunEnding { + Completed, + Interrupted, + Raised, +} diff --git a/crates/sqlbuild-state/src/runs/tests.rs b/crates/sqlbuild-state/src/runs/tests.rs new file mode 100644 index 0000000000..d53d8384e6 --- /dev/null +++ b/crates/sqlbuild-state/src/runs/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_run_status.rs"] +mod run_status; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/runs/tests/test_run_status.rs b/crates/sqlbuild-state/src/runs/tests/test_run_status.rs new file mode 100644 index 0000000000..e8a2edcc9a --- /dev/null +++ b/crates/sqlbuild-state/src/runs/tests/test_run_status.rs @@ -0,0 +1,81 @@ +use crate::runs::main::run_status::run_status; +use crate::runs::models::{RunEnding, RunNodeCounts}; +use crate::runs::tests::test_types::RunStatusTestCase; + +#[test] +fn given_node_counts_when_finishing_a_run_then_the_status_follows_the_server_rule() { + let test_cases = [ + RunStatusTestCase { + description: "every node succeeded, one with a warning", + counts: RunNodeCounts { + succeeded: 3, + warned: 1, + ..RunNodeCounts::default() + }, + ending: RunEnding::Completed, + expected_status: "succeeded", + }, + RunStatusTestCase { + description: "a failure beside successes is partial", + counts: RunNodeCounts { + succeeded: 2, + failed: 1, + ..RunNodeCounts::default() + }, + ending: RunEnding::Completed, + expected_status: "partial", + }, + RunStatusTestCase { + description: "nodes skipped after the only node failed is a failure", + counts: RunNodeCounts { + failed: 1, + skipped: 2, + ..RunNodeCounts::default() + }, + ending: RunEnding::Completed, + expected_status: "failed", + }, + RunStatusTestCase { + description: "planned nodes that never started count against the run", + counts: RunNodeCounts { + succeeded: 1, + not_started: 2, + ..RunNodeCounts::default() + }, + ending: RunEnding::Completed, + expected_status: "partial", + }, + RunStatusTestCase { + description: "an interrupt cancels, whatever ran", + counts: RunNodeCounts { + succeeded: 1, + ..RunNodeCounts::default() + }, + ending: RunEnding::Interrupted, + expected_status: "cancelled", + }, + RunStatusTestCase { + description: "a run that stopped on an error failed, whatever ran", + counts: RunNodeCounts { + succeeded: 4, + ..RunNodeCounts::default() + }, + ending: RunEnding::Raised, + expected_status: "failed", + }, + RunStatusTestCase { + description: "a run with no nodes succeeded", + counts: RunNodeCounts::default(), + ending: RunEnding::Completed, + expected_status: "succeeded", + }, + ]; + for test_case in test_cases { + assert_eq!( + run_status(&test_case.counts, test_case.ending), + test_case.expected_status, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/runs/tests/test_types.rs b/crates/sqlbuild-state/src/runs/tests/test_types.rs new file mode 100644 index 0000000000..6d71eb2c5a --- /dev/null +++ b/crates/sqlbuild-state/src/runs/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::runs::models::{RunEnding, RunNodeCounts}; + +pub(crate) struct RunStatusTestCase { + pub(crate) description: &'static str, + pub(crate) counts: RunNodeCounts, + pub(crate) ending: RunEnding, + pub(crate) expected_status: &'static str, +} diff --git a/fensu.toml b/fensu.toml index de2eb450d0..6058ce6bab 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2077,7 +2077,7 @@ paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/cla reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs", "crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs"] +paths = ["crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs", "crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs", "crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs"] reason = "Each state table has its own Python-facing create and prune bindings. The DDL and table naming live in the shared fixed_state_table_sql and state_table_name; each prune keeps its own retention rule." [[dupes.allowlist]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 68a98b4599..5ccea0bc09 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -19,6 +19,7 @@ Online: https://sqlbuild.com/docs/cli/build/ - Failure behavior - Limits - Fingerprints +- Run records - Runtime artifacts Compiles, plans, and executes the selected build lifecycle, running the full selected scope. Use @@ -342,6 +343,21 @@ After a timeout, SQLBuild checks the connection and replaces it if it is no long After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. A build writes a row only for a resource whose identity differs from its latest row, so a build that changes no definitions writes none. In a project with model migration history, every build writes a row for each resource it builds. See [Planning and Change Detection](../concepts/planning.md) for details. +## Run records + +Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` in +the target schema before its first node runs, and a `finished` row with its status and node counts +when it ends. A run with only a `started` row was killed or crashed. A target whose schema is +`preserve` has no schema of its own, so its runs are not recorded. + +Each row records the command, its selection as typed, the jobs it names, the target, the +warehouse user, the `sqb login` identity if you're signed in, the operating-system user, the +machine name, the SQLBuild version and the Git commit. Of `--vars` it records only the names, +never the values. Set `[settings] record_run_host = false` to leave out the machine name. + +Recording a run never fails the build: if a row can't be written, the build prints a warning and +carries on. + ## Runtime artifacts Build writes executed lifecycle SQL to `target/run/models/`. These files contain the actual SQL that was executed, including resolved cursor bounds and runtime substitutions. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 63c1f46108..57d1c97021 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -521,6 +521,7 @@ delta_compile = true | `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](python-nodes/providers.md#secrets). | | `allow_warehouse_round_trips` | `false` | Allow a loader to depend on a task or asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](python-nodes/overview.md#warehouse-round-trips). | | `ignore_audit_failures` | `false` | Let builds go on past failing audits: they still run and record `fail`, but promote, apply and build downstream as if they passed. A target, a job and `--ignore-audit-failures` / `--no-ignore-audit-failures` override it. See [Ignoring audit failures](audits.md#ignoring-audit-failures). | +| `record_run_host` | `true` | Record the machine name in each run's rows in `_sqlbuild_runs`. See [Run records](../cli/build.md#run-records). | ### Unit test and scenario limits diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7ab8e77d84..cb48515dc1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -191,6 +191,7 @@ OLD_NAME_VIEW_TABLE_NAME: str MICROBATCH_TABLE_NAME: str NODE_RESULTS_TABLE_NAME: str OBSERVATIONS_TABLE_NAME: str +RUNS_TABLE_NAME: str AUDIT_RESULTS_TABLE_NAME: str JANITOR_EVENTS_TABLE_NAME: str COLUMN_NODE_TYPE: str @@ -5541,6 +5542,13 @@ def build_observations_create_table_sql( render_qualified_name: Callable[..., str | None], render_framework_type: Callable[[FrameworkType], str], ) -> str: ... +def build_runs_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... def build_audit_results_prune_sql( *, database: str | None, @@ -12095,6 +12103,7 @@ class SettingsConfig(NativeRecord): allow_missing_secrets: bool allow_warehouse_round_trips: bool ignore_audit_failures: bool + record_run_host: bool def __init__( self, sql_analysis: bool = ..., @@ -12117,6 +12126,7 @@ class SettingsConfig(NativeRecord): allow_missing_secrets: bool = ..., allow_warehouse_round_trips: bool = ..., ignore_audit_failures: bool = ..., + record_run_host: bool = ..., ) -> None: ... @property def sql_validation(self) -> bool: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 12e9f34c50..143a9e7c76 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -27,6 +27,7 @@ HistoricalTimestampSnapshotSql, ) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin +from sqlbuild.adapter.contract.classes.runs_table import RunsTableMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -104,7 +105,7 @@ ) -class BaseAdapter(RetentionAdapterMixin, StrictAdapter): +class BaseAdapter(RetentionAdapterMixin, RunsTableMixin, StrictAdapter): """Adapter base with ANSI SQL defaults.""" adapter_name: ClassVar[str] diff --git a/src/sqlbuild/adapter/contract/classes/runs_table.py b/src/sqlbuild/adapter/contract/classes/runs_table.py new file mode 100644 index 0000000000..81936fca81 --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/runs_table.py @@ -0,0 +1,27 @@ +"""Adapter implementations for the runs table every build and seed appends to.""" + +from __future__ import annotations + +from typing import Any, cast + + +class RunsTableMixin: + """Provide runs-table DDL and the warehouse identity a run records.""" + + def render_current_warehouse_user(self) -> str | None: + """Render the session's warehouse identity as a SQL expression; None without one.""" + + return None + + def render_create_runs_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the runs table when it is missing.""" + + from sqlbuild._native import build_runs_create_table_sql + + adapter: Any = cast(Any, self) + return build_runs_create_table_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 589a6b5478..dbf477190b 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1083,6 +1083,9 @@ def render_current_timestamp(self) -> str: def render_current_utc_timestamp(self) -> str: return "CURRENT_TIMESTAMP()" + def render_current_warehouse_user(self) -> str | None: + return "SESSION_USER()" + def render_loader_logical_type( self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 ) -> str: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 48dc9308b3..5f71843f45 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -708,6 +708,9 @@ def render_current_timestamp(self) -> str: def render_current_utc_timestamp(self) -> str: return "now64(6, 'UTC')" + def render_current_warehouse_user(self) -> str | None: + return "currentUser()" + def normalize_row_diff_numeric_type(self, column_type: str) -> str | None: return normalize_numeric_family(type_sql=canonical_type(column_type), dialect=None) @@ -1970,6 +1973,11 @@ def render_create_observations_table_sql(self, *, database: str | None, schema: super().render_create_observations_table_sql(database=database, schema=schema) ) + def render_create_runs_table_sql(self, *, database: str | None, schema: str) -> str: + return _state_table_ddl( + super().render_create_runs_table_sql(database=database, schema=schema) + ) + def render_create_node_result_table_sql(self, *, database: str | None, schema: str) -> str: return _state_table_ddl( super().render_create_node_result_table_sql(database=database, schema=schema) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 2696200e7d..a137aeaf7c 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -814,6 +814,9 @@ def render_current_timestamp(self) -> str: def render_current_utc_timestamp(self) -> str: return "current_timestamp()" + def render_current_warehouse_user(self) -> str | None: + return "current_user()" + def sql_analysis_dialect(self) -> str | None: """Return the SQL analysis dialect name for this adapter, if any.""" diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 13e44efa93..9b056b7636 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1002,6 +1002,9 @@ def render_current_timestamp(self) -> str: def render_current_utc_timestamp(self) -> str: return "(now() AT TIME ZONE 'UTC')" + def render_current_warehouse_user(self) -> str | None: + return "CURRENT_USER" + def create_table_as( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 3317a3cb44..65d20ccf5d 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -692,6 +692,13 @@ def render_create_resource_costs_table_sql(self, *, database: str | None, schema schema=schema, ) + def render_create_runs_table_sql(self, *, database: str | None, schema: str) -> str: + return self._with_state_table_retention( + create_sql=super().render_create_runs_table_sql(database=database, schema=schema), + database=database, + schema=schema, + ) + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: return self._with_state_table_retention( create_sql=super().render_create_microbatch_state_table_sql( @@ -1301,6 +1308,9 @@ def render_current_timestamp(self) -> str: def render_current_utc_timestamp(self) -> str: return "SYSDATE()" + def render_current_warehouse_user(self) -> str | None: + return "CURRENT_USER()" + def create_table_as( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 274308f4df..199b7c7770 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -525,6 +525,15 @@ def render_create_observations_table_sql(self, *, database: str | None, schema: key_columns=("kind", "node"), ) + def render_create_runs_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import RUNS_TABLE_NAME + + return self._keyed_state_table_sql( + create_sql=super().render_create_runs_table_sql(database=database, schema=schema), + table=(database, schema, RUNS_TABLE_NAME), + key_columns=("run_id", "phase"), + ) + def render_create_consumption_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: @@ -2307,6 +2316,9 @@ def render_current_timestamp(self) -> str: def render_current_utc_timestamp(self) -> str: return "SYSUTCDATETIME()" + def render_current_warehouse_user(self) -> str | None: + return "SUSER_SNAME()" + def render_delete_insert( self, *, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 79694159b2..82c8bef1ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -77,13 +77,14 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( expected_stdout_fragments=( "relations to archive 1", "archives to delete 0", - "objects skipped 5", + "objects skipped 6", "main.janitor_tracked_extra -> main._sqb_archive__", "main.janitor_untracked_extra relation is not tracked by SQLBuild", "main.partition_state relation matches exclude pattern 'partition_*'", "main._sqlbuild_fingerprints relation matches exclude pattern", "main._sqlbuild_microbatches relation matches exclude pattern", "main._sqlbuild_consumption relation matches exclude pattern", + "main._sqlbuild_runs relation matches exclude pattern", "Archived 1 relation.", ), expected_existing_tables=( @@ -94,6 +95,7 @@ def test_given_default_config_when_running_janitor_then_it_reports_disabled( "_sqlbuild_microbatches", "_sqlbuild_janitor_events", "_sqlbuild_consumption", + "_sqlbuild_runs", ), expected_missing_tables=("janitor_tracked_extra",), expected_archived_original_names=("janitor_tracked_extra",), @@ -317,7 +319,7 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ expected_exit_code=0, expected_stdout_fragments=( "relations to archive 2", - "objects skipped 3", + "objects skipped 4", "main.__sqb_a13f09c2e7b8__model__daily_revenue -> main._sqb_archive__", "main.__sqb_a13f09c2e7b8__source__raw_orders -> main._sqb_archive__", "main.__sqb_a13f09c2e7b__model__daily_revenue relation is not tracked by SQLBuild", diff --git a/tests/goldens/compiler/failure/local-config-unknown-key.json b/tests/goldens/compiler/failure/local-config-unknown-key.json index 345adf1548..da518bfee3 100644 --- a/tests/goldens/compiler/failure/local-config-unknown-key.json +++ b/tests/goldens/compiler/failure/local-config-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: allow_missing_secrets, allow_warehouse_round_trips, auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, ignore_audit_failures, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", + "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: allow_missing_secrets, allow_warehouse_round_trips, auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, ignore_audit_failures, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, record_run_host, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", "help": "did you mean 'sql_analysis'?" } ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 79e06bd725..a07f1a708e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1436,3 +1436,42 @@ class JobSelectorBuildTestCase: expected_exit_code: int expected_tables: frozenset[str] expected_fragment: str = "" + + +@dataclass(frozen=True) +class RunRecordTestCase: + description: str + build_args: tuple[str, ...] + expected_exit_code: int + # Each row as (phase, command, select_json, named_jobs_json, var_keys_json, status, counts). + expected_rows: list[tuple[str, str, str, str, str, str | None, str | None]] + settings: str = "" + expected_host_recorded: bool = True + + +@dataclass(frozen=True) +class RunRecordFailureTestCase: + description: str + warehouse_setup: str + expected_exit_code: int + expected_warning: str + expected_tables: frozenset[str] + + +@dataclass(frozen=True) +class InterruptedRunRecordTestCase: + description: str + expected_tables: frozenset[str] + + +@dataclass(frozen=True) +class KilledRunRecordTestCase: + description: str + expected_phases: list[str] + + +@dataclass(frozen=True) +class SeedRunRecordTestCase: + description: str + # Each row as (phase, command, select_json, named_jobs_json, var_keys_json, status, counts). + expected_rows: list[tuple[str, str, str, str, str, str | None, str | None]] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index d9a0ab9067..0be7c535d1 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1373,3 +1373,85 @@ def write_python_rules_project(*, project_dir: Path, fixture: str, rules_toml: s "models/orders.sql": 'MODEL (description "Orders");\nSELECT 1 AS order_id\n', }, ) + + +RUN_RECORD_COLUMNS: str = ( + "phase, command, select_json, named_jobs_json, var_keys_json, status, node_counts_json" +) + +RUN_RECORD_PROJECT_FILES: dict[str, str] = { + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\nSELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/broken_orders.sql": ( + "MODEL (description 'Fails at run time.', materialized table);\n" + "SELECT CAST('x' AS INTEGER) AS order_id\n" + ), + "orchestration/jobs/nightly_job.toml": ( + 'name = "nightly_job"\ndescription = "Orders."\nselect = "+orders"\n' + ), + "python/crash.py": ( + '"""A task that kills its process, as a crash or kill -9 would."""\n\n' + "import os\n\n" + "from sqlbuild.tasks import task\n\n\n" + "@task()\n" + "def crash(ctx):\n" + ' """End the process without unwinding."""\n\n' + " os._exit(137)\n" + ), + "python/fail.py": ( + '"""A task that raises, so the build exits non-zero."""\n\n' + "from sqlbuild.tasks import task\n\n\n" + "@task()\n" + "def fail(ctx):\n" + ' """Raise."""\n\n' + ' raise RuntimeError("task failed")\n' + ), +} + + +def write_run_record_project(*, project_dir: Path, settings: str) -> None: + """Write the project, with `settings` appended to the project file.""" + + files: dict[str, str] = { + **RUN_RECORD_PROJECT_FILES, + "sqlbuild_project.toml": ( + 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n' + settings + ), + } + for relative_path, contents in files.items(): + destination: Path = project_dir / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(contents, encoding="utf-8") + + +def run_rows(*, project_dir: Path, columns: str) -> list[tuple[object, ...]]: + """`columns` of every runs-table row, oldest first.""" + + with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: + return warehouse.sql( + f"SELECT {columns} FROM main._sqlbuild_runs ORDER BY recorded_at, phase DESC" + ).fetchall() + + +def built_tables(*, project_dir: Path) -> frozenset[str]: + """The project's own tables in the warehouse, without SQLBuild's state tables.""" + + with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: + rows: list[tuple[str]] = warehouse.sql( + "SELECT table_name FROM duckdb_tables() " + "WHERE NOT internal AND NOT starts_with(table_name, '_sqlbuild')" + ).fetchall() + return frozenset(name for (name,) in rows) + + +def prepare_warehouse(*, project_dir: Path, sql: str) -> None: + """Run `sql` in the project's warehouse before the build.""" + + with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: + warehouse.execute(sql) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py new file mode 100644 index 0000000000..3a20c3a7ad --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py @@ -0,0 +1,312 @@ +"""Every build records its start and end in the target's runs table, and a failed write never fails it.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from sqlbuild.adapter.contract.classes.runs_table import RunsTableMixin +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + InterruptedRunRecordTestCase, + KilledRunRecordTestCase, + RunRecordFailureTestCase, + RunRecordTestCase, + SeedRunRecordTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + RUN_RECORD_COLUMNS, + built_tables, + prepare_warehouse, + run_rows, + write_run_record_project, +) + +_SUCCEEDED_TWO: str = '{"succeeded":2,"warned":0,"failed":0,"skipped":0,"not_started":0}' + + +@pytest.mark.parametrize( + "test_case", + [ + RunRecordTestCase( + description="a job build records its selection, the job and only the names of its vars", + build_args=( + "--select", + "job:nightly_job", + "--vars", + '{"api_token": "s3cret-value", "batch": 2}', + ), + expected_exit_code=0, + expected_rows=[ + ( + "started", + "build", + '["job:nightly_job"]', + '["nightly_job"]', + '["api_token","batch"]', + None, + None, + ), + ( + "finished", + "build", + '["job:nightly_job"]', + '["nightly_job"]', + '["api_token","batch"]', + "succeeded", + _SUCCEEDED_TWO, + ), + ], + ), + RunRecordTestCase( + description="a bare job name counts as a named job", + build_args=("--select", "nightly_job"), + expected_exit_code=0, + expected_rows=[ + ("started", "build", '["nightly_job"]', '["nightly_job"]', "[]", None, None), + ( + "finished", + "build", + '["nightly_job"]', + '["nightly_job"]', + "[]", + "succeeded", + _SUCCEEDED_TWO, + ), + ], + ), + RunRecordTestCase( + description="a failing model beside successes finishes partial", + build_args=("--select", "stg_orders", "broken_orders"), + expected_exit_code=1, + expected_rows=[ + ("started", "build", '["stg_orders","broken_orders"]', "[]", "[]", None, None), + ( + "finished", + "build", + '["stg_orders","broken_orders"]', + "[]", + "[]", + "partial", + '{"succeeded":1,"warned":0,"failed":1,"skipped":0,"not_started":0}', + ), + ], + ), + RunRecordTestCase( + description="a failing task fails the run even though no model failed", + build_args=("--select", "task:fail"), + expected_exit_code=1, + expected_rows=[ + ("started", "build", '["task:fail"]', "[]", "[]", None, None), + ( + "finished", + "build", + '["task:fail"]', + "[]", + "[]", + "failed", + '{"succeeded":0,"warned":0,"failed":1,"skipped":0,"not_started":0}', + ), + ], + ), + RunRecordTestCase( + description="a project that turns off the host leaves it out", + build_args=("--select", "stg_orders"), + expected_exit_code=0, + expected_rows=[ + ("started", "build", '["stg_orders"]', "[]", "[]", None, None), + ( + "finished", + "build", + '["stg_orders"]', + "[]", + "[]", + "succeeded", + '{"succeeded":1,"warned":0,"failed":0,"skipped":0,"not_started":0}', + ), + ], + settings="\n[settings]\nrecord_run_host = false\n", + expected_host_recorded=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_when_it_runs_then_it_records_its_start_and_end( + test_case: RunRecordTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_run_record_project(project_dir=tmp_path, settings=test_case.settings) + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "build", "--no-tests", *test_case.build_args] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, captured + assert run_rows(project_dir=tmp_path, columns=RUN_RECORD_COLUMNS) == test_case.expected_rows + identities: list[tuple[object, ...]] = run_rows( + project_dir=tmp_path, + columns="run_id, attempt, retry_of_run_id, orchestration_run_id, target_name, " + "warehouse_user, os_user IS NOT NULL, host IS NOT NULL, sqlbuild_version IS NOT NULL", + ) + assert len({identity[0] for identity in identities}) == 1 + assert {identity[1:] for identity in identities} == { + (1, None, None, "dev", None, True, test_case.expected_host_recorded, True) + } + assert "s3cret-value" not in str(run_rows(project_dir=tmp_path, columns="*")) + + +@pytest.mark.parametrize( + "test_case", + [ + SeedRunRecordTestCase( + description="a seed run records its start and end like a build", + expected_rows=[ + ("started", "seed", "[]", "[]", "[]", None, None), + ( + "finished", + "seed", + "[]", + "[]", + "[]", + "succeeded", + '{"succeeded":1,"warned":0,"failed":0,"skipped":0,"not_started":0}', + ), + ], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_seed_run_when_it_loads_then_it_records_its_start_and_end( + test_case: SeedRunRecordTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_run_record_project(project_dir=tmp_path, settings="") + (tmp_path / "seeds").mkdir() + (tmp_path / "seeds" / "regions.csv").write_text("region_id\n1\n", encoding="utf-8") + (tmp_path / "seeds" / "regions.yml").write_text( + "seeds:\n - name: regions\n description: Regions.\n columns:\n" + " - name: region_id\n type: INTEGER\n", + encoding="utf-8", + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", "seed"]) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == 0, captured + assert run_rows(project_dir=tmp_path, columns=RUN_RECORD_COLUMNS) == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + RunRecordFailureTestCase( + description="a view in the table's place makes every write fail; the build still succeeds", + warehouse_setup="CREATE VIEW main._sqlbuild_runs AS SELECT 1 AS run_id", + expected_exit_code=0, + expected_warning="Warning: could not record that run", + expected_tables=frozenset({"stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unwritable_runs_table_when_building_then_it_warns_and_succeeds( + test_case: RunRecordFailureTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_run_record_project(project_dir=tmp_path, settings="") + prepare_warehouse(project_dir=tmp_path, sql=test_case.warehouse_setup) + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "build", "--no-tests", "-s", "stg_orders"] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, captured + assert captured.count(test_case.expected_warning) == 2, captured + assert built_tables(project_dir=tmp_path) == test_case.expected_tables + + +@pytest.mark.parametrize( + "test_case", + [ + InterruptedRunRecordTestCase( + description="Ctrl+C while the started row is written stops the build before any node", + expected_tables=frozenset(), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_interrupt_while_recording_when_building_then_the_build_stops( + test_case: InterruptedRunRecordTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_run_record_project(project_dir=tmp_path, settings="") + + def interrupt(self: object) -> str | None: + raise KeyboardInterrupt + + monkeypatch.setattr(RunsTableMixin, "render_current_warehouse_user", interrupt) + + with pytest.raises(KeyboardInterrupt): + main( + [ + "--project-dir", + str(tmp_path), + "--no-color", + "build", + "--no-tests", + "-s", + "stg_orders", + ] + ) + + assert built_tables(project_dir=tmp_path) == test_case.expected_tables + + +@pytest.mark.parametrize( + "test_case", + [ + KilledRunRecordTestCase( + description="a build killed mid-run leaves only its started row", + expected_phases=["started"], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_killed_build_when_reading_its_runs_then_only_started_remains( + test_case: KilledRunRecordTestCase, + tmp_path: Path, +) -> None: + write_run_record_project(project_dir=tmp_path, settings="") + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(tmp_path), + "--no-color", + "build", + "--no-tests", + "--select", + "task:crash", + ], + capture_output=True, + text=True, + timeout=300, + env={**os.environ, "NO_COLOR": "1"}, + check=False, + ) + + assert completed.returncode == 137, completed.stdout + completed.stderr + assert [row[0] for row in run_rows(project_dir=tmp_path, columns="phase")] == ( + test_case.expected_phases + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 4d306ad30b..57a4eabdeb 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -111,7 +111,7 @@ class WorkerExecutionTestCase: model_sql: str warehouse_query: str expected_status: str - expected_rows: list[tuple[int] | tuple[str]] + expected_rows: list[tuple[str | int | bool | None, ...]] expected_node_count: int = 1 # Node counts by status when the nodes don't all share the run's status. expected_node_counts: dict[str, int] | None = None diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 64da65119a..a2f2f71aec 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -792,6 +792,17 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ @pytest.mark.parametrize( "test_case", [ + WorkerExecutionTestCase( + "a scheduled run records itself in the runs table as the CLI does, with the server run", + "SELECT 5 AS amount", + "SELECT phase, command, orchestration_run_id IS NOT NULL, sqlbuild_user, attempt, " + "retry_of_run_id, status FROM main._sqlbuild_runs ORDER BY recorded_at, phase DESC", + "success", + [ + ("started", "build", True, "system:schedule", 1, None, None), + ("finished", "build", True, "system:schedule", 1, None, "succeeded"), + ], + ), WorkerExecutionTestCase( "successful model writes actual rows", "WITH source(value) AS (VALUES ('23')) SELECT CAST(value AS BIGINT) AS amount FROM source", diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index f675a0fd21..0fb684cdf3 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -447,3 +447,9 @@ class SnowflakeKeyedDeleteInsertFailureTestCase: description: str failing_keyword: str expected_prefixes: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeWarehouseUserTestCase: + description: str + expected_expression: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py index 2991ef7135..a90c918377 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_state_table_ddl.py @@ -14,6 +14,7 @@ SnowflakeStateTableFallbackTestCase, SnowflakeStateTableRetentionErrorTestCase, SnowflakeStateTableSchemaFallbackTestCase, + SnowflakeWarehouseUserTestCase, ) from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( CustomInitSnowflakeAdapter, @@ -94,6 +95,12 @@ expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_microbatches (", expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", ), + SnowflakeStateTableDdlTestCase( + description="runs", + render_method="render_create_runs_table_sql", + expected_prefix="CREATE TABLE IF NOT EXISTS analytics.marts._sqlbuild_runs (", + expected_suffix=") DATA_RETENTION_TIME_IN_DAYS = 90", + ), ], ids=lambda case: case.description, ) @@ -440,3 +447,19 @@ def test_given_one_schema_rejecting_90_days_when_creating_state_tables_then_othe ) == test_case.expected_executed ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeWarehouseUserTestCase( + description="the session's Snowflake user", + expected_expression="CURRENT_USER()", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_snowflake_session_when_recording_a_run_then_its_user_comes_from_the_warehouse( + test_case: SnowflakeWarehouseUserTestCase, +) -> None: + assert SnowflakeAdapter().render_current_warehouse_user() == test_case.expected_expression diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index ec6578fac4..fc13b958db 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -326,6 +326,21 @@ After a timeout, SQLBuild checks the connection and replaces it if it is no long After a successful build, SQLBuild writes version identities to `_sqlbuild_fingerprints` in the target schema. These are used on subsequent runs to detect changes and skip unchanged work. A build writes a row only for a resource whose identity differs from its latest row, so a build that changes no definitions writes none. In a project with model migration history, every build writes a row for each resource it builds. See [Planning and Change Detection](/docs/concepts/planning) for details. +## Run records + +Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` in +the target schema before its first node runs, and a `finished` row with its status and node counts +when it ends. A run with only a `started` row was killed or crashed. A target whose schema is +`preserve` has no schema of its own, so its runs are not recorded. + +Each row records the command, its selection as typed, the jobs it names, the target, the +warehouse user, the `sqb login` identity if you're signed in, the operating-system user, the +machine name, the SQLBuild version and the Git commit. Of `--vars` it records only the names, +never the values. Set `[settings] record_run_host = false` to leave out the machine name. + +Recording a run never fails the build: if a row can't be written, the build prints a warning and +carries on. + ## Runtime artifacts Build writes executed lifecycle SQL to `target/run/models/`. These files contain the actual SQL that was executed, including resolved cursor bounds and runtime substitutions. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 908c9fd342..693363e381 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -495,6 +495,7 @@ delta_compile = true | `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](/docs/concepts/python-nodes/providers#secrets). | | `allow_warehouse_round_trips` | `false` | Allow a loader to depend on a task or asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](/docs/concepts/python-nodes/overview#warehouse-round-trips). | | `ignore_audit_failures` | `false` | Let builds go on past failing audits: they still run and record `fail`, but promote, apply and build downstream as if they passed. A target, a job and `--ignore-audit-failures` / `--no-ignore-audit-failures` override it. See [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures). | +| `record_run_host` | `true` | Record the machine name in each run's rows in `_sqlbuild_runs`. See [Run records](/docs/cli/build#run-records). | ### Unit test and scenario limits