diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 7a68e32d8a..2d6591fd76 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -23,7 +23,7 @@ pub(crate) const UNSUPPORTED_WAREHOUSE_CODE: &str = "C260"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; -/// The warehouse group each CLI command selects a target's default warehouse from. +/// The warehouse group each CLI command uses; a location's metadata syncs use the cheap `sync` group. pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("audit", Some("build")), ("build", Some("build")), @@ -44,13 +44,13 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("janitor", Some("query")), ("scenario", Some("query")), ("test", Some("query")), + ("location", Some("sync")), ("clean", None), ("compile", None), ("dag", None), ("format", None), ("init", None), ("lineage", None), - ("location", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 99ae5b143c..ad8150d254 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -279,7 +279,7 @@ pub(crate) const EVENT_EXPORT_SEVERITIES: [&str; 4] = ["debug", "info", "warning /// `TIME_TRAVEL_RETENTION_MATERIALIZATIONS` in declared order. pub(crate) const RETENTION_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; pub(crate) const MATERIALIZATION_DEFAULT_KEYS: [&str; 2] = ["time_travel_retention", "table_type"]; -pub(crate) const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"]; +pub(crate) const WAREHOUSE_GROUPS: [&str; 3] = ["build", "query", "sync"]; pub(crate) const SNAPSHOT_FULL_REFRESH_POLICIES: [&str; 3] = ["allow", "deny", "require_confirmation"]; pub(crate) const SNAPSHOT_SCHEMA_CHANGE_POLICIES: [&str; 3] = diff --git a/crates/sqlbuild-core/src/cost_command/constants.rs b/crates/sqlbuild-core/src/cost_command/constants.rs index 83f1bbc19b..f9b76b7636 100644 --- a/crates/sqlbuild-core/src/cost_command/constants.rs +++ b/crates/sqlbuild-core/src/cost_command/constants.rs @@ -27,3 +27,12 @@ pub const MISSING_METRIC_STATUSES: [&str; 4] = /// Statuses a delayed refresh may still improve. pub const REFRESHABLE_STATUSES: [&str; 2] = ["pending", "partial"]; pub const LEDGER_PERSISTENCE_FAILURE_PREFIX: &str = "Statement-ledger persistence failed ("; +/// The limitation of a run whose statements query history still showed running, which a re-read can complete. +pub const INCOMPLETE_HISTORY_LIMITATION: &str = + "One or more run statements were still incomplete in query history."; +/// The limitation of a run whose recent statements had no warehouse metadata yet, which a re-read can complete. +pub const MISSING_METADATA_LIMITATION: &str = + "One or more run statements had missing Snowflake warehouse metadata."; +/// The limitations a re-read of a run that saw every statement can still lift. +pub const CHANGEABLE_LIMITATIONS: [&str; 2] = + [INCOMPLETE_HISTORY_LIMITATION, MISSING_METADATA_LIMITATION]; diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index b9d86c1362..63f2aa3e7b 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -342,7 +342,7 @@ const E_EVENT_EXPORT_SEVERITY: EnumSpec = EnumSpec { const E_WAREHOUSE_GROUP: EnumSpec = EnumSpec { name: "WarehouseGroup", doc: "Command group that selects a target's default warehouse.", - members: &[("BUILD", "build"), ("QUERY", "query")], + members: &[("BUILD", "build"), ("QUERY", "query"), ("SYNC", "sync")], }; const E_SOURCE_WRITE_STRATEGY: EnumSpec = EnumSpec { name: "SourceWriteStrategy", diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index d3fdf60b51..c041a0efc2 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -102,6 +102,7 @@ const R_TARGET_WAREHOUSES_CONFIG: RecordSpec = RecordSpec { fields: &[ FieldSpec::new("build", FieldDefault::None), FieldSpec::new("query", FieldDefault::None), + FieldSpec::new("sync", FieldDefault::None), ], }; const R_TARGET_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs index 19a5cafb28..b41b92014b 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs @@ -19,7 +19,7 @@ use sqlbuild_python_base::records::types::ErrorId; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; /// `WarehouseGroup` members in declaration order. -const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"]; +const WAREHOUSE_GROUPS: [&str; 3] = ["build", "query", "sync"]; fn project_config_error(py: Python<'_>, message: &str, help: Option<&str>) -> PyErr { new_error(py, ErrorId::ProjectConfig, message, (None, help)) diff --git a/crates/sqlbuild-python-front/src/native_host/main/elapsed_seconds.rs b/crates/sqlbuild-python-front/src/native_host/main/elapsed_seconds.rs new file mode 100644 index 0000000000..c8951e68ee --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/elapsed_seconds.rs @@ -0,0 +1,8 @@ +//! Elapsed time between two Python datetimes. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// Seconds from `from` to `to`, negative when `to` is earlier. +pub fn elapsed_seconds(from: &Bound<'_, PyAny>, to: &Bound<'_, PyAny>) -> PyResult { + to.sub(from)?.call_method0("total_seconds")?.extract() +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/mod.rs b/crates/sqlbuild-python-front/src/native_host/main/mod.rs index 4cb7f09303..393249f52a 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/mod.rs @@ -1,5 +1,6 @@ pub mod append_observations; pub mod append_replaced_consumption; +pub mod elapsed_seconds; pub mod execute; pub mod fixed_state_table_sql; pub mod integer_literal; @@ -9,4 +10,6 @@ pub mod recorder_callbacks; pub mod register_audit_results_state; pub mod register_consumption_state; pub mod register_observations_state; +pub mod reports_cost_capability; +pub mod seconds_since; pub mod state_table_name; diff --git a/crates/sqlbuild-python-front/src/native_host/main/reports_cost_capability.rs b/crates/sqlbuild-python-front/src/native_host/main/reports_cost_capability.rs new file mode 100644 index 0000000000..4b42260f7c --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/reports_cost_capability.rs @@ -0,0 +1,15 @@ +//! Whether a cost-aware adapter reports a cost capability. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// Whether `adapter` reports a cost capability; a failing report still counts as one. +pub fn reports_cost_capability(adapter: &Bound<'_, PyAny>) -> PyResult { + let none: Bound<'_, PyAny> = + host_entry(adapter.py(), ExecutorHost::CostCapability)?.getattr("NONE")?; + Ok(match adapter.call_method0("cost_capability") { + Ok(capability) => !capability.eq(none)?, + Err(_) => true, + }) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs b/crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs new file mode 100644 index 0000000000..f4bce84a3e --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs @@ -0,0 +1,14 @@ +//! Time since a Python datetime, by the UTC clock now. + +use crate::cursor::constants::DATETIME_MODULE; +use crate::native_host::main::elapsed_seconds::elapsed_seconds; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// Seconds since `value`, by the UTC clock now. +pub fn seconds_since(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let now: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")? + .call_method1("now", (imported(py, DATETIME_MODULE, "UTC")?,))?; + elapsed_seconds(value, &now) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/_helpers/cost_run_records.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/cost_run_records.rs new file mode 100644 index 0000000000..ca95cb231f --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/cost_run_records.rs @@ -0,0 +1,134 @@ +//! Plain cost runs read from `CostRunRecord` objects, and their fields built back into Python values. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_core::cost_report::models::{CostResource, CostSummary}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +pub(crate) fn text(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + native_field(value, field)?.extract() +} + +pub(crate) fn optional_text(value: &Bound<'_, PyAny>, field: &str) -> PyResult> { + native_field(value, field)?.extract() +} + +/// `str(value.)`, the text a `Decimal` field is stored as. +pub(crate) fn decimal(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + Ok(native_field(value, field)?.str()?.to_string()) +} + +pub(crate) fn integer(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + native_field(value, field)?.extract() +} + +pub(crate) fn timestamp(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + native_field(value, field)? + .call_method0("isoformat")? + .extract() +} + +pub(crate) fn resource(value: &Bound<'_, PyAny>) -> PyResult { + Ok(CostResource { + resource_type: text(value, "resource_type")?, + resource_name: text(value, "resource_name")?, + warehouse_name: text(value, "warehouse_name")?, + warehouse_size: text(value, "warehouse_size")?, + warehouse_type: optional_text(value, "warehouse_type")?, + cluster_number: native_field(value, "cluster_number")?.extract()?, + query_count: integer(value, "query_count")?, + attributed_seconds: decimal(value, "attributed_seconds")?, + bytes_scanned: integer(value, "bytes_scanned")?, + estimated_compute_credits: decimal(value, "estimated_compute_credits")?, + estimated_usd: decimal(value, "estimated_usd")?, + }) +} + +/// Python constructors of the record fields. +pub(crate) struct Builders<'py> { + pub(crate) decimal: Bound<'py, PyAny>, + pub(crate) timestamp: Bound<'py, PyAny>, +} + +impl<'py> Builders<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + decimal: imported(py, "decimal", "Decimal")?, + timestamp: imported(py, "datetime", "datetime")?.getattr("fromisoformat")?, + }) + } + + pub(crate) fn decimal(&self, text: &str) -> PyResult> { + self.decimal.call1((text,)) + } +} + +pub(crate) fn resource_object<'py>( + builders: &Builders<'py>, + resource: &CostResource, +) -> PyResult> { + let py: Python<'py> = builders.decimal.py(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("resource_type", &resource.resource_type)?; + fields.set_item("resource_name", &resource.resource_name)?; + fields.set_item("warehouse_name", &resource.warehouse_name)?; + fields.set_item("warehouse_size", &resource.warehouse_size)?; + fields.set_item("warehouse_type", &resource.warehouse_type)?; + fields.set_item("cluster_number", resource.cluster_number)?; + fields.set_item("query_count", resource.query_count)?; + fields.set_item( + "attributed_seconds", + builders.decimal(&resource.attributed_seconds)?, + )?; + fields.set_item("bytes_scanned", resource.bytes_scanned)?; + fields.set_item( + "estimated_compute_credits", + builders.decimal(&resource.estimated_compute_credits)?, + )?; + fields.set_item("estimated_usd", builders.decimal(&resource.estimated_usd)?)?; + new_record(py, RecordId::ResourceCost, (), Some(&fields)) +} + +pub(crate) fn summary_object<'py>( + builders: &Builders<'py>, + cost: &CostSummary, +) -> PyResult> { + let py: Python<'py> = builders.decimal.py(); + let mut resources: Vec> = Vec::with_capacity(cost.resources.len()); + for resource in &cost.resources { + resources.push(resource_object(builders, resource)?); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item( + "status", + host_entry(py, ExecutorHost::CostStatus)?.call1((cost.status.as_str(),))?, + )?; + fields.set_item("usd_per_credit", builders.decimal(&cost.usd_per_credit)?)?; + fields.set_item("rate_source", &cost.rate_source)?; + fields.set_item("resources", PyTuple::new(py, resources)?)?; + fields.set_item("query_count", cost.query_count)?; + fields.set_item( + "attributed_seconds", + builders.decimal(&cost.attributed_seconds)?, + )?; + fields.set_item("bytes_scanned", cost.bytes_scanned)?; + fields.set_item( + "estimated_compute_credits", + builders.decimal(&cost.estimated_compute_credits)?, + )?; + fields.set_item("estimated_usd", builders.decimal(&cost.estimated_usd)?)?; + fields.set_item("expected_statement_count", cost.expected_statement_count)?; + fields.set_item("observed_statement_count", cost.observed_statement_count)?; + fields.set_item("missing_statement_count", cost.missing_statement_count)?; + fields.set_item("skipped_statement_count", cost.skipped_statement_count)?; + fields.set_item("source", &cost.source)?; + fields.set_item("method", &cost.method)?; + fields.set_item("limitations", PyTuple::new(py, &cost.limitations)?)?; + fields.set_item("message", &cost.message)?; + host_entry(py, ExecutorHost::RunCostSummary)?.call((), Some(&fields)) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs index 7d7aae7125..4367bf553f 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod cost_run_records; pub(crate) mod resource_costs_state; diff --git a/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs index 347a1823b2..50a1ae3831 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs @@ -31,8 +31,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; const CONFIGURED_RATE_SOURCE: &str = "configured"; /// The adapter method that marks the resource costs table as supported. pub(crate) const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql"; -/// Collection budget of each re-read of an older run, which a build waits for. -pub(crate) const REFRESH_BUDGET_SECONDS: f64 = 20.0; /// Create-if-missing DDL for the resource costs table, with adapter-rendered column types. #[pyfunction] @@ -210,9 +208,14 @@ pub(crate) fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { let limitations: Vec = serde_json::from_str(&text(7)?) .map_err(|error| PyValueError::new_err(format!("unreadable limitations: {error}")))?; let run_started_at: String = utc_text(&cell(9)?)?; - let started_epoch: i64 = DateTime::parse_from_rfc3339(&run_started_at) - .map_err(|error| PyValueError::new_err(error.to_string()))? - .timestamp(); + let epoch = |text: &str| -> PyResult { + Ok(DateTime::parse_from_rfc3339(text) + .map_err(|error| PyValueError::new_err(error.to_string()))? + .timestamp()) + }; + let started_epoch: i64 = epoch(&run_started_at)?; + let run_completed_at: String = utc_text(&cell(10)?)?; + let completed_epoch: i64 = epoch(&run_completed_at)?; Ok(RunCostAttempt { run_id: text(0)?, orchestration_run_id: optional_text(&cell(1)?)?, @@ -226,7 +229,8 @@ pub(crate) fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { }, target_name: optional_text(&cell(8)?)?, run_started_at, - run_completed_at: utc_text(&cell(10)?)?, + run_completed_at, started_epoch, + completed_epoch, }) } diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/artifact_error.rs b/crates/sqlbuild-python-front/src/resource_costs/main/artifact_error.rs new file mode 100644 index 0000000000..2017c6bfc2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/artifact_error.rs @@ -0,0 +1,16 @@ +//! The `CostArtifactError` Python raised for a cost artifact. + +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; +use sqlbuild_core::cost_report::errors::CostArtifactError; +use sqlbuild_python_base::records::main::registry::native_attr::native_attr; +use sqlbuild_python_base::records::types::NativeAttr; + +/// The `CostArtifactError` Python raised for the same artifact. +pub fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr { + match native_attr(py, NativeAttr::CostArtifactError) + .and_then(|class| class.call1((error.message.as_str(),))) + { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs b/crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs new file mode 100644 index 0000000000..239ae71867 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs @@ -0,0 +1,25 @@ +//! A `CostRunRecord` as a plain run. + +use crate::resource_costs::_helpers::cost_run_records::integer; +use crate::resource_costs::_helpers::cost_run_records::optional_text; +use crate::resource_costs::_helpers::cost_run_records::text; +use crate::resource_costs::_helpers::cost_run_records::timestamp; +use crate::resource_costs::main::summary::summary; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_report::models::CostRun; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A `CostRunRecord` as a plain run. +pub fn cost_run(record: &Bound<'_, PyAny>) -> PyResult { + Ok(CostRun { + run_id: text(record, "run_id")?, + adapter_name: text(record, "adapter_name")?, + target_name: optional_text(record, "target_name")?, + build_status: text(record, "build_status")?, + started_at: timestamp(record, "started_at")?, + completed_at: timestamp(record, "completed_at")?, + cost: summary(&native_field(record, "cost")?)?, + version: integer(record, "version")?, + had_executable_work: native_field(record, "had_executable_work")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs b/crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs new file mode 100644 index 0000000000..7ba02d5fee --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs @@ -0,0 +1,14 @@ +//! Every `CostRunRecord` of an iterable as a plain run. + +use crate::resource_costs::main::cost_run::cost_run; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_report::models::CostRun; + +/// Every record of an iterable as a plain run. +pub fn cost_runs(records: &Bound<'_, PyAny>) -> PyResult> { + let mut runs: Vec = Vec::new(); + for record in records.try_iter()? { + runs.push(cost_run(&record?)?); + } + Ok(runs) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs b/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs index 14fe99725c..5da7f95174 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs @@ -1 +1,7 @@ +pub mod artifact_error; +pub mod cost_run; +pub mod cost_runs; +pub mod record_object; +pub mod record_tuple; pub mod register_resource_costs_state; +pub mod summary; diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/record_object.rs b/crates/sqlbuild-python-front/src/resource_costs/main/record_object.rs new file mode 100644 index 0000000000..f991d0996d --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/record_object.rs @@ -0,0 +1,31 @@ +//! A plain run as the `CostRunRecord` it was read from. + +use crate::resource_costs::_helpers::cost_run_records::Builders; +use crate::resource_costs::_helpers::cost_run_records::summary_object; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::cost_report::models::CostRun; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// A plain run as the `CostRunRecord` it was read from. +pub fn record_object<'py>(py: Python<'py>, run: &CostRun) -> PyResult> { + let builders: Builders<'py> = Builders::new(py)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("version", run.version)?; + fields.set_item("run_id", &run.run_id)?; + fields.set_item("adapter_name", &run.adapter_name)?; + fields.set_item("target_name", &run.target_name)?; + fields.set_item("build_status", &run.build_status)?; + fields.set_item( + "started_at", + builders.timestamp.call1((run.started_at.as_str(),))?, + )?; + fields.set_item( + "completed_at", + builders.timestamp.call1((run.completed_at.as_str(),))?, + )?; + fields.set_item("cost", summary_object(&builders, &run.cost)?)?; + fields.set_item("had_executable_work", run.had_executable_work)?; + new_record(py, RecordId::CostRunRecord, (), Some(&fields)) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/record_tuple.rs b/crates/sqlbuild-python-front/src/resource_costs/main/record_tuple.rs new file mode 100644 index 0000000000..1e1baa2ce7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/record_tuple.rs @@ -0,0 +1,15 @@ +//! Records of several plain runs, as a tuple. + +use crate::resource_costs::main::record_object::record_object; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_core::cost_report::models::CostRun; + +/// Records of several plain runs, as a tuple. +pub fn record_tuple<'py>(py: Python<'py>, runs: &[CostRun]) -> PyResult> { + let mut records: Vec> = Vec::with_capacity(runs.len()); + for run in runs { + records.push(record_object(py, run)?); + } + PyTuple::new(py, records) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/summary.rs b/crates/sqlbuild-python-front/src/resource_costs/main/summary.rs new file mode 100644 index 0000000000..84dbb5cf30 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/summary.rs @@ -0,0 +1,41 @@ +//! A `RunCostSummary` as a plain cost summary. + +use crate::resource_costs::_helpers::cost_run_records::decimal; +use crate::resource_costs::_helpers::cost_run_records::integer; +use crate::resource_costs::_helpers::cost_run_records::optional_text; +use crate::resource_costs::_helpers::cost_run_records::resource; +use crate::resource_costs::_helpers::cost_run_records::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_report::models::{CostResource, CostSummary}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A `RunCostSummary` as a plain cost summary. +pub fn summary(value: &Bound<'_, PyAny>) -> PyResult { + let mut resources: Vec = Vec::new(); + for item in native_field(value, "resources")?.try_iter()? { + resources.push(resource(&item?)?); + } + let mut limitations: Vec = Vec::new(); + for item in native_field(value, "limitations")?.try_iter()? { + limitations.push(item?.extract()?); + } + Ok(CostSummary { + status: native_field(value, "status")?.getattr("value")?.extract()?, + usd_per_credit: decimal(value, "usd_per_credit")?, + rate_source: text(value, "rate_source")?, + resources, + query_count: integer(value, "query_count")?, + attributed_seconds: decimal(value, "attributed_seconds")?, + bytes_scanned: integer(value, "bytes_scanned")?, + estimated_compute_credits: decimal(value, "estimated_compute_credits")?, + estimated_usd: decimal(value, "estimated_usd")?, + expected_statement_count: integer(value, "expected_statement_count")?, + observed_statement_count: integer(value, "observed_statement_count")?, + missing_statement_count: integer(value, "missing_statement_count")?, + skipped_statement_count: integer(value, "skipped_statement_count")?, + source: text(value, "source")?, + method: text(value, "method")?, + limitations, + message: optional_text(value, "message")?, + }) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/models.rs b/crates/sqlbuild-python-front/src/resource_costs/models.rs index 6fb02bc785..5bed834d05 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/models.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/models.rs @@ -4,7 +4,6 @@ use crate::native_host::constants::ROWS_PER_STATEMENT; use crate::native_host::main::execute::execute; use crate::native_host::models::Location; use crate::resource_costs::_helpers::resource_costs_state::CREATE_TABLE_METHOD; -use crate::resource_costs::_helpers::resource_costs_state::REFRESH_BUDGET_SECONDS; use crate::resource_costs::_helpers::resource_costs_state::observation_cells; use crate::resource_costs::_helpers::resource_costs_state::run_attempt; use crate::resource_costs::_helpers::resource_costs_state::table; @@ -130,11 +129,12 @@ impl<'py> ResourceCostStore<'py> { .collect() } - /// Re-collect one older run through the adapter, priced at `rate`. + /// Re-collect one older run through the adapter, priced at `rate`, within `budget_seconds`. pub fn recollect( &self, run: &RunCostAttempt, (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + budget_seconds: f64, ) -> PyResult> { let py: Python<'py> = self.adapter.py(); let parse = imported(py, "datetime", "datetime")?.getattr("fromisoformat")?; @@ -165,7 +165,7 @@ impl<'py> ResourceCostStore<'py> { ("rate_source", rate_source.clone()), ( "budget_seconds", - PyFloat::new(py, REFRESH_BUDGET_SECONDS).into_any(), + PyFloat::new(py, budget_seconds).into_any(), ), ], ) diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs index 78824e1588..e8d0910251 100644 --- a/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs +++ b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs @@ -12,6 +12,9 @@ use sqlbuild_analysis::snowflake_cost::models::QueryTag; use sqlbuild_core::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; use sqlbuild_core::cost_allocation::main::query_history_sql::query_history_sql; use sqlbuild_core::cost_allocation::models::{QueryHistoryWindow, RunAllocation}; +use sqlbuild_core::cost_command::constants::{ + INCOMPLETE_HISTORY_LIMITATION, MISSING_METADATA_LIMITATION, +}; use sqlbuild_host::entries::models::ExecutorHost; use crate::cost_telemetry::_helpers::allocation::SummaryState; @@ -29,6 +32,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::native_host::main::elapsed_seconds::elapsed_seconds; +use sqlbuild_python_front::native_host::main::seconds_since::seconds_since; /// Default budget of one collection, which the at-exit collection keeps short. const DEADLINE_SECONDS: f64 = 30.0; @@ -40,6 +45,8 @@ const STATEMENT_TIMEOUT_CODE: &str = "000630"; const RUNNING_STATUS: &str = "RUNNING"; const HOOK_RESOURCE_TYPE: &str = "hook"; const ROW_WIDTH: usize = 14; +/// Seconds after which a finished query still without a warehouse size, such as DDL, used none. +const METADATA_SETTLE_SECONDS: f64 = 120.0; const NO_WAREHOUSE_COMPUTE: &str = "no_warehouse_compute"; const MISSING_WAREHOUSE_METADATA: &str = "missing_warehouse_metadata"; const INCOMPLETE_HISTORY: &str = "incomplete_history"; @@ -311,7 +318,13 @@ fn row_observation<'py>( return Ok(Err(INCOMPLETE_HISTORY)); } if milliseconds.eq(0)? || warehouse_name.is_none() || warehouse_size.is_none() { - return Ok(Err(MISSING_WAREHOUSE_METADATA)); + let settled: bool = + !running && elapsed_seconds(&end_time, &observed_at)? >= METADATA_SETTLE_SECONDS; + return Ok(Err(if settled { + NO_WAREHOUSE_COMPUTE + } else { + MISSING_WAREHOUSE_METADATA + })); } let tag: Option = if query_tag.is_instance_of::() { sqlbuild_query_tag(&query_tag.extract::()?) @@ -392,7 +405,7 @@ fn open_rows<'py>( )) } -/// Observations of the run window: one open read, halved windows when saturated or timed out. +/// Observations of the run window: an open read while it is open, else only the window itself. fn query_observations<'py>( history: &History<'py>, (started_at, completed_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -402,7 +415,11 @@ fn query_observations<'py>( let skew: Bound<'py, PyAny> = timedelta(py, ("minutes", 10))?; let window_start: Bound<'py, PyAny> = started_at.sub(&skew)?; let window_end: Bound<'py, PyAny> = completed_at.add(&skew)?; - let (rows, saturated) = open_rows(history, (&window_start, &window_end))?; + let (rows, saturated) = if seconds_since(&window_end)?.is_sign_positive() { + bounded_rows(history, (&window_start, &window_end))? + } else { + open_rows(history, (&window_start, &window_end))? + }; let observations: Bound<'py, PyList> = PyList::empty(py); let mut observed: HashSet = HashSet::new(); let mut skipped: Vec<(String, &'static str)> = Vec::new(); @@ -531,12 +548,10 @@ fn with_observation_gaps( let incomplete: bool = relevant.contains(&INCOMPLETE_HISTORY); let mut added: Vec = Vec::new(); if missing_metadata { - added.push( - "One or more run statements had missing Snowflake warehouse metadata.".to_owned(), - ); + added.push(MISSING_METADATA_LIMITATION.to_owned()); } if incomplete { - added.push("One or more run statements were still incomplete in query history.".to_owned()); + added.push(INCOMPLETE_HISTORY_LIMITATION.to_owned()); } summary.state.limitations = with_limitations(&summary.state.limitations, added); let query_count: usize = summary @@ -556,6 +571,15 @@ fn with_observation_gaps( summary } +/// Whether every expected statement is visible and finished, lacking at most settling metadata. +fn awaits_only_metadata(expected: &HashSet, observed: &Observed<'_>) -> bool { + expected.is_subset(&observed.observed) + && !observed + .skipped + .iter() + .any(|(id, reason)| expected.contains(id) && *reason == INCOMPLETE_HISTORY) +} + fn incomplete_observation(expected: &HashSet, observed: &Observed<'_>) -> bool { observed.skipped.iter().any(|(id, reason)| { expected.contains(id) @@ -690,12 +714,12 @@ fn collect<'py>( return Ok(with_ledger_failure(complete, failure.as_deref())); } summary = Some(current); - if attempt + 1 < collection.attempts { - if history.deadline - monotonic(py)? <= collection.retry_delay_seconds { - break; - } - imported(py, "time", "sleep")?.call1((collection.retry_delay_seconds,))?; + let last: bool = + attempt + 1 >= collection.attempts || awaits_only_metadata(&expected_ids, &observed); + if last || history.deadline - monotonic(py)? <= collection.retry_delay_seconds { + break; } + imported(py, "time", "sleep")?.call1((collection.retry_delay_seconds,))?; } let mut summary: Summary = summary.unwrap_or_else(|| partial("Snowflake query history could not be collected.")); diff --git a/crates/sqlbuild-python-project/src/native_host/models.rs b/crates/sqlbuild-python-project/src/native_host/models.rs index 415afdd71a..70a2b87d14 100644 --- a/crates/sqlbuild-python-project/src/native_host/models.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -97,3 +97,30 @@ pub struct CalendarRequest { pub timezone: String, pub trigger: TriggerDefinition, } + +/// A worker build's cost finalization, held until the run has published its outcome. +#[derive(Debug, Default)] +pub struct DeferredCost(std::sync::Mutex>>); + +impl DeferredCost { + const POISONED: &str = "deferred cost lock is poisoned"; + + /// Keep `finalization`, replacing any earlier one. + pub fn defer(&self, finalization: &Bound<'_, PyAny>) -> PyResult<()> { + *self + .0 + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err(Self::POISONED))? = + Some(finalization.clone().unbind()); + Ok(()) + } + + /// The deferred finalization, once. + pub fn take(&self) -> PyResult>> { + Ok(self + .0 + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err(Self::POISONED))? + .take()) + } +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/models.rs b/crates/sqlbuild-python-project/src/project_snapshot/models.rs index 7ec951f026..385f0b8903 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/models.rs @@ -5,6 +5,7 @@ use crate::compiled_project::constants::REPORT_ATTEMPTS; use crate::compiled_project::constants::SKIP_ACTION; use crate::compiled_project::constants::SQL_TEST_KEY_PREFIX; use crate::compiled_project::constants::STORE_BUSY_STATUS; +use crate::native_host::models::DeferredCost; use crate::project_snapshot::_helpers::snapshot::snapshot_execution::Completion; use crate::project_snapshot::_helpers::snapshot::snapshot_execution::is_function; use crate::project_snapshot::_helpers::snapshot::snapshot_execution::poisoned; @@ -142,6 +143,8 @@ pub struct WorkerBuild { refusal: Mutex>, /// Error-severity audit failures the build let through, reported before the run finishes. ignored_audit_failures: AtomicU32, + /// The build's cost finalization, collected once the run has reported its outcome. + deferred_cost: DeferredCost, } impl WorkerBuild { @@ -164,6 +167,7 @@ impl WorkerBuild { returned: AtomicBool::new(false), ignored_audit_failures: AtomicU32::new(0), refusal: Mutex::default(), + deferred_cost: DeferredCost::default(), }); for key in state.execution.skipped_nodes.keys() { state.complete(worker.py(), key, (NodeOutcome::Skipped, None, (None, None)))?; @@ -180,6 +184,16 @@ impl WorkerBuild { self.execution.claim.request.run_id } + /// Keep the build's cost finalization to collect after the run's outcome is published. + pub fn defer_cost(&self, finalization: &Bound<'_, PyAny>) -> PyResult<()> { + self.deferred_cost.defer(finalization) + } + + /// The cost finalization the build deferred, once. + pub fn take_deferred_cost(&self) -> PyResult>> { + self.deferred_cost.take() + } + /// The build let `count` error-severity audit failures through. pub fn ignored_audit_failures(&self, count: i64) { self.ignored_audit_failures.store( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 60837a78cc..c7e486bdbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -17,7 +17,9 @@ use crate::bindings::_helpers::build_command::execution::Preparation; use crate::bindings::_helpers::build_command::execution::write_run_context; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::check::check_results::check_result_text; +use crate::bindings::_helpers::cost_report::degradation::RUN_END_SWEEP_SECONDS; use crate::bindings::_helpers::cost_report::degradation::finalize_build_cost as finalize_cost; +use crate::bindings::_helpers::cost_report::degradation::finalize_build_cost_within as finalize_cost_within; use crate::bindings::_helpers::cost_report::degradation::render_build_cost; use crate::bindings::_helpers::cost_report::functions::cost_output_payload; use sqlbuild_core::build_progress::models::PythonNodeText; @@ -37,6 +39,8 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::plan_has_executable_work::plan_has_executable_work; use sqlbuild_python_compile::compile_outputs::main::write_python_check_runtime_target::write_python_check_runtime_target; use sqlbuild_python_compile::compile_outputs::main::write_runtime_target::write_runtime_target; +use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; +use sqlbuild_python_front::native_host::main::reports_cost_capability::reports_cost_capability; use sqlbuild_python_output::build_progress::main::build_result_text::build_result_text; use sqlbuild_python_output::build_progress::main::format_build_footer_with::format_build_footer_with; use sqlbuild_python_output::result_output::main::results::format_build_execution_json::format_build_execution_json; @@ -48,6 +52,8 @@ const CHECK_KIND: &str = "check"; const CHECK_WARNED: &str = "check warned"; const PENDING_MESSAGE: &str = "Build cost collection has not completed."; const INTERRUPTED_MESSAGE: &str = "Build was interrupted before cost collection completed."; +const DEFERRED_MESSAGE: &str = "Warehouse costs are collected after the run finishes."; +const COLLECTING_LINE: &str = "Collecting warehouse costs...\n"; /// How one cost finalization differs from a completed collection. pub(crate) struct CostRequest<'py> { @@ -62,6 +68,15 @@ pub(crate) fn finalize_build_cost<'py>( context: &BuildContext<'py>, pipeline: &Bound<'py, PyAny>, cost: CostRequest<'py>, +) -> PyResult> { + finalize_cost(&cost_finalization(context, pipeline, cost)?) +} + +/// The `BuildCostFinalization` of one cost request. +fn cost_finalization<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + cost: CostRequest<'py>, ) -> PyResult> { let py: Python<'py> = context.py(); let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; @@ -103,9 +118,56 @@ pub(crate) fn finalize_build_cost<'py>( )); } fields.extend(cost.overrides); - let finalization: Bound<'py, PyAny> = - new_record_with(py, RecordId::BuildCostFinalization, &fields)?; - finalize_cost(&finalization) + new_record_with(py, RecordId::BuildCostFinalization, &fields) +} + +/// A finished build's cost: a worker's waits until its run has finished; the CLI sweeps briefly. +fn finished_build_cost<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + (status, started_at, completed_at): (&'static str, &Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = context.py(); + let request = |overrides: Vec<(&'static str, Bound<'py, PyAny>)>| CostRequest { + status, + started_at: started_at.clone(), + completed_at: completed_at.clone(), + overrides, + }; + let adapter: Bound<'py, PyAny> = context.get("adapter")?; + if !cost_aware(&adapter)? || !reports_cost_capability(&adapter)? { + return finalize_build_cost( + context, + pipeline, + request(vec![("render", boolean(py, false))]), + ); + } + if let Some(worker) = &context.worker { + worker.defer_cost(&cost_finalization( + context, + pipeline, + request(vec![("render", boolean(py, false))]), + )?)?; + return finalize_build_cost( + context, + pipeline, + request(vec![ + ("collect", boolean(py, false)), + ("render", boolean(py, false)), + ("cost_status", cost_status(py, "PENDING")?), + ("cost_message", text(py, DEFERRED_MESSAGE)), + ]), + ); + } + emit(&context.stream, COLLECTING_LINE)?; + finalize_cost_within( + &cost_finalization( + context, + pipeline, + request(vec![("render", boolean(py, false))]), + )?, + Some(RUN_END_SWEEP_SECONDS), + ) } fn cost_status<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -384,23 +446,6 @@ pub(crate) fn complete_direct_build<'py>( joined(&context.get("effective_project_dir")?, &["target"])?; write_runtime_target(&target_dir, &plan, result)?; write_python_check_runtime_target(&target_dir, check_results)?; - let cost_started: Instant = Instant::now(); - let record: PyResult> = finalize_build_cost( - context, - pipeline, - CostRequest { - status: if exit_code == 0 { - SUCCESS_STATUS - } else { - "failed" - }, - started_at: started_at.clone(), - completed_at, - overrides: vec![("render", boolean(py, false))], - }, - ); - let cost_seconds: f64 = finish_cost_timing(context, cost_started); - let record: Bound<'py, PyAny> = record?; let checks: Vec = check_results .try_iter()? .map(|check| Ok(check_node_text(&check_result_text(&check?)?))) @@ -415,6 +460,22 @@ pub(crate) fn complete_direct_build<'py>( checks, )?; emit(&context.stream, &format!("\n{footer}\n"))?; + let cost_started: Instant = Instant::now(); + let record: PyResult> = finished_build_cost( + context, + pipeline, + ( + if exit_code == 0 { + SUCCESS_STATUS + } else { + "failed" + }, + started_at, + completed_at, + ), + ); + let cost_seconds: f64 = finish_cost_timing(context, cost_started); + let record: Bound<'py, PyAny> = record?; let payload: Bound<'py, PyAny> = format_build_execution_json_with( py, result, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 194c02db2e..8636c86891 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -35,6 +35,7 @@ use crate::bindings::_helpers::cli::termination::{ use sqlbuild_python_compile::compile_command::constants::COMPILE_COMMAND_NAME; use sqlbuild_python_compile::compile_command::main::invocation::format_compile_error_json::format_compile_error_json; use crate::bindings::_helpers::cost_report::attribution::take_cost_status; +use crate::bindings::_helpers::cost_report::deferred::collect_deferred_cost_after_finish; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use sqlbuild_python_base::native_host::models::InterruptHandler; use sqlbuild_python_base::native_host::main::interrupted::interrupted; @@ -933,6 +934,7 @@ fn execute_parsed_invocation<'py>( py, &format!("Worker execution completed: {}\n", outcome.as_str()), )?; + collect_deferred_cost_after_finish(py, &worker); if !matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { dispatched?; finished?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 20515e7b7a..52ccd14ab5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -34,9 +34,17 @@ use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_ use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; -/// The `WarehouseGroup` whose warehouse is a target's `warehouses.build`; any other reads `query`. const BUILD_GROUP: &str = "build"; const QUERY_GROUP: &str = "query"; +const SYNC_GROUP: &str = "sync"; +/// The target warehouses each group reads, in order, before the connection's own warehouse. +const GROUP_WAREHOUSES: [(&str, &[&str]); 3] = [ + (BUILD_GROUP, &[BUILD_GROUP]), + (QUERY_GROUP, &[QUERY_GROUP]), + (SYNC_GROUP, &[SYNC_GROUP, QUERY_GROUP]), +]; +/// The target warehouse a sync falls back to only when the connection names none, so it can still run. +const SYNC_LAST_RESORT: &str = BUILD_GROUP; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( &[ @@ -115,15 +123,10 @@ fn resolved_warehouse<'py>( ) } -fn group_value(target: &Bound<'_, PyAny>, group_name: &str) -> PyResult> { +fn group_value(target: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { if target.is_none() { return Ok(None); } - let attribute: &str = if group_name == BUILD_GROUP { - BUILD_GROUP - } else { - QUERY_GROUP - }; optional_text(&native_field( &native_field(target, "warehouses")?, attribute, @@ -135,8 +138,27 @@ pub(crate) fn find_authored( inputs: WarehouseInputs<'_, '_>, group: &Bound<'_, PyAny>, ) -> PyResult> { - let group_name: String = native_field(group, "value")?.extract()?; - authored_group(inputs, &group_name) + authored_group(inputs, &group_name(group)?) +} + +/// The name of a command group, such as `sync`. +fn group_name(group: &Bound<'_, PyAny>) -> PyResult { + native_field(group, "value")?.extract() +} + +/// The target warehouse `group` uses before the connection's; a sync uses `build` only when the connection names none. +fn group_authored( + inputs: WarehouseInputs<'_, '_>, + group: &Bound<'_, PyAny>, + connection_config: &Bound<'_, PyAny>, +) -> PyResult> { + if let Some(found) = find_authored(inputs, group)? { + return Ok(Some(found)); + } + if group_name(group)? == SYNC_GROUP && connection_warehouse(connection_config)?.is_none() { + return authored_group(inputs, SYNC_LAST_RESORT); + } + Ok(None) } /// The active target's authored, unexpanded warehouse for the group named `group_name`. @@ -163,11 +185,20 @@ fn authored_group<'py>( let project_target: Bound<'py, PyAny> = native_field(&project_config, "targets")?.call_method1("get", (&target_name,))?; let target_name: String = target_name.extract()?; - Ok(authored_warehouse( - (&target_name, group_name), - group_value(&local_target, group_name)?, - group_value(&project_target, group_name)?, - )) + let attributes: &[&str] = GROUP_WAREHOUSES + .iter() + .find(|(group, _)| *group == group_name) + .map_or(&[QUERY_GROUP], |(_, attributes)| attributes); + for attribute in attributes { + if let Some(found) = authored_warehouse( + (&target_name, attribute), + group_value(&local_target, attribute)?, + group_value(&project_target, attribute)?, + ) { + return Ok(Some(found)); + } + } + Ok(None) } /// The group warehouse with `${...}` templates expanded, checked as an identifier. @@ -236,7 +267,7 @@ pub(crate) fn resolve_warehouse( } if let Some(inputs) = inputs && !group.is_none() - && let Some(found) = find_authored(inputs, group)? + && let Some(found) = group_authored(inputs, group, connection_config)? { let warehouse: String = expand_authored(&found, inputs, cli_vars)?; return Ok((Some(warehouse), found.source())); @@ -247,14 +278,14 @@ pub(crate) fn resolve_warehouse( }) } -/// Every warehouse the target configures: its `build` and `query` group warehouses and its connection warehouse. +/// Every warehouse the target configures: its `build`, `query` and `sync` group warehouses and its connection warehouse. pub(crate) fn configured_warehouses( inputs: WarehouseInputs<'_, '_>, connection_config: &Bound<'_, PyAny>, ) -> PyResult> { let none: Bound<'_, PyAny> = none(connection_config.py()); let mut names: std::collections::BTreeSet = std::collections::BTreeSet::new(); - for group_name in [BUILD_GROUP, QUERY_GROUP] { + for group_name in [BUILD_GROUP, QUERY_GROUP, SYNC_GROUP] { if let Some(authored) = authored_group(inputs, group_name)? { names.insert(expand_authored(&authored, inputs, &none)?); } @@ -328,7 +359,9 @@ pub(crate) fn apply_warehouse<'py>( let py: Python<'py> = config.py(); let scope: CommandScope<'py> = command_scope(py)?; let authored: bool = match inputs { - Some(inputs) if !scope.group.is_none() => find_authored(inputs, &scope.group)?.is_some(), + Some(inputs) if !scope.group.is_none() => { + group_authored(inputs, &scope.group, config)?.is_some() + } _ => false, }; if scope.cli_warehouse.is_none() && !authored { @@ -398,7 +431,12 @@ pub(crate) fn describe_connection_warehouse<'py>( let found: Option = if scope.group.is_none() { None } else { - find_authored(inputs, &scope.group)? + match find_authored(inputs, &scope.group)? { + None if group_name(&scope.group)? == SYNC_GROUP => { + authored_group(inputs, SYNC_LAST_RESORT)? + } + found => found, + } }; if let Some(found) = found { let expands_to: bool = match expand_authored(&found, inputs, &scope.cli_vars) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/deferred.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/deferred.rs new file mode 100644 index 0000000000..46627a3ea9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/deferred.rs @@ -0,0 +1,25 @@ +//! A worker build's warehouse costs, collected after the run reports its outcome so the run is not held open. + +use pyo3::prelude::{PyResult, Python}; + +use crate::bindings::_helpers::cost_report::degradation::finalize_build_cost_within; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; + +/// Collect a deferred build's costs after its run finished; a failure only warns, as the outcome is published. +pub(crate) fn collect_deferred_cost_after_finish(py: Python<'_>, worker: &WorkerBuild) { + if let Err(error) = collect_deferred_cost(py, worker) { + let _ = write_stderr( + py, + &format!("warning: warehouse cost collection failed: {error}\n"), + ); + } +} + +fn collect_deferred_cost(py: Python<'_>, worker: &WorkerBuild) -> PyResult<()> { + let Some(finalization) = worker.take_deferred_cost()? else { + return Ok(()); + }; + write_stderr(py, "Collecting warehouse costs...\n")?; + finalize_build_cost_within(finalization.bind(py), None).map(|_| ()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 39ea17272d..a89ee28dfd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -14,7 +14,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::cost_report::attribution::report_cost_status; use crate::bindings::_helpers::cost_report::publication::publish_resource_costs; -use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -22,6 +21,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; +use sqlbuild_python_front::resource_costs::main::cost_run::cost_run; const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; @@ -30,6 +30,8 @@ const CONFIGURED_RATE_SOURCE: &str = "configured"; const STATEMENT_LEDGER_PATH: [&str; 4] = ["target", "executions", "", "statements.jsonl"]; const BUILD_BREAKDOWN_LIMIT: usize = 5; const FALLBACK_TERMINAL_COLUMNS: usize = 100; +/// Query-history budget of a command-line build's run-end sweep; what it misses, a later build reads. +pub(crate) const RUN_END_SWEEP_SECONDS: f64 = 6.0; /// Log the failure with its traceback, as `_LOGGER.exception` inside the handler did. fn log_failure(py: Python<'_>, message: &str, error: &PyErr) -> PyResult<()> { @@ -271,7 +273,7 @@ fn collection_failure<'py>( fn collected_summary<'py>( finalization: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, - (config, rate_source): (&Bound<'py, PyAny>, &str), + ((config, rate_source), budget): ((&Bound<'py, PyAny>, &str), Option), ) -> PyResult> { let py: Python<'py> = finalization.py(); let run_id: Bound<'py, PyAny> = native_field(finalization, "run_id")?; @@ -300,6 +302,10 @@ fn collected_summary<'py>( ("statement_ledger_path", ledger), ("usd_per_credit", native_field(config, "usd_per_credit")?), ("rate_source", PyString::new(py, rate_source).into_any()), + ( + "budget_seconds", + pyo3::IntoPyObjectExt::into_bound_py_any(budget, py)?, + ), ], ) } @@ -308,6 +314,14 @@ fn collected_summary<'py>( #[pyfunction] pub(crate) fn finalize_build_cost<'py>( finalization: &Bound<'py, PyAny>, +) -> PyResult> { + finalize_build_cost_within(finalization, None) +} + +/// [`finalize_build_cost`] with the collection's query-history budget, or the adapter's default. +pub(crate) fn finalize_build_cost_within<'py>( + finalization: &Bound<'py, PyAny>, + budget: Option, ) -> PyResult> { let py: Python<'py> = finalization.py(); let adapter: Bound<'py, PyAny> = native_field(finalization, "adapter")?; @@ -331,7 +345,7 @@ pub(crate) fn finalize_build_cost<'py>( return Ok(py.None().into_bound(py)); } let cost: Bound<'py, PyAny> = if native_field(finalization, "collect")?.is_truthy()? { - match collected_summary(finalization, &adapter, rate) { + match collected_summary(finalization, &adapter, (rate, budget)) { Ok(summary) => summary, Err(error) => return collection_failure(finalization, rate, "collection", &error), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs index f6145f6d98..c36ac70df1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs @@ -13,11 +13,12 @@ use sqlbuild_core::cost_report::main::format_cost_history::format_cost_history a use sqlbuild_core::cost_report::main::select_cost_run::select_cost_run; use sqlbuild_core::cost_report::models::{BreakdownOptions, CostRun}; -use crate::bindings::_helpers::cost_report::records::{ - cost_run, cost_runs, record_object, record_tuple, -}; use crate::bindings::_helpers::cost_report::store; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_front::resource_costs::main::cost_run::cost_run; +use sqlbuild_python_front::resource_costs::main::cost_runs::cost_runs; +use sqlbuild_python_front::resource_costs::main::record_object::record_object; +use sqlbuild_python_front::resource_costs::main::record_tuple::record_tuple; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; const BREAKDOWN_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index b7f6602189..d5d28ef528 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -1,8 +1,8 @@ pub(crate) mod attribution; +pub(crate) mod deferred; pub(crate) mod degradation; pub(crate) mod exception_types; pub(crate) mod functions; pub(crate) mod publication; -pub(crate) mod records; pub(crate) mod registration; pub(crate) mod store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs index 34a11bf9c2..74c03b1b62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs @@ -1,6 +1,6 @@ //! Record a build's run cost in the target's state schema, then re-read older incomplete runs. -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -11,13 +11,19 @@ use sqlbuild_state::resource_costs::constants::COLLECTION_FAILED_STATUS; use sqlbuild_state::resource_costs::main::refresh_candidates::refresh_candidates; use sqlbuild_state::resource_costs::models::RunCostAttempt; -use crate::bindings::_helpers::cost_report::records::summary; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::resource_costs::main::summary::summary; use sqlbuild_python_front::resource_costs::models::CostRunIdentity; use sqlbuild_python_front::resource_costs::models::ResourceCostStore; /// The attempt number of a build's own collection; each accepted re-read appends the next. const FIRST_ATTEMPT: i64 = 1; +/// Query-history time a build spends re-reading older runs, across all of them. +const REFRESH_TOTAL_SECONDS: f64 = 8.0; +/// Query-history time one older run's re-read may take. +const REFRESH_RUN_SECONDS: f64 = 5.0; +/// Remaining time below which no further older run is re-read. +const REFRESH_MIN_SECONDS: f64 = 1.0; fn coverage(summary: &CostSummary) -> CostCoverage { CostCoverage { @@ -47,9 +53,14 @@ fn refresh_older( .into_iter() .filter(|latest| latest.run_id != run_id) .collect(); + let started: Instant = Instant::now(); for older in refresh_candidates(&latest, unix_now()?) { + let remaining: f64 = REFRESH_TOTAL_SECONDS - started.elapsed().as_secs_f64(); + if remaining < REFRESH_MIN_SECONDS { + break; + } let Ok(refreshed) = store - .recollect(&older, rate) + .recollect(&older, rate, remaining.min(REFRESH_RUN_SECONDS)) .and_then(|collected| summary(&collected)) else { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs deleted file mode 100644 index 647bf3805b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! `CostRunRecord` objects read into plain cost runs, and plain runs built back into records. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use sqlbuild_core::cost_report::errors::CostArtifactError; -use sqlbuild_core::cost_report::models::{CostResource, CostRun, CostSummary}; -use sqlbuild_host::entries::models::ExecutorHost; - -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::registry::native_attr::native_attr; -use sqlbuild_python_base::records::types::NativeAttr; -use sqlbuild_python_base::records::types::RecordId; - -/// The `CostArtifactError` Python raised for the same artifact. -pub(crate) fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr { - match native_attr(py, NativeAttr::CostArtifactError) - .and_then(|class| class.call1((error.message.as_str(),))) - { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - -fn text(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - native_field(value, field)?.extract() -} - -fn optional_text(value: &Bound<'_, PyAny>, field: &str) -> PyResult> { - native_field(value, field)?.extract() -} - -/// `str(value.)`, the text a `Decimal` field is stored as. -fn decimal(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - Ok(native_field(value, field)?.str()?.to_string()) -} - -fn integer(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - native_field(value, field)?.extract() -} - -fn timestamp(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - native_field(value, field)? - .call_method0("isoformat")? - .extract() -} - -fn resource(value: &Bound<'_, PyAny>) -> PyResult { - Ok(CostResource { - resource_type: text(value, "resource_type")?, - resource_name: text(value, "resource_name")?, - warehouse_name: text(value, "warehouse_name")?, - warehouse_size: text(value, "warehouse_size")?, - warehouse_type: optional_text(value, "warehouse_type")?, - cluster_number: native_field(value, "cluster_number")?.extract()?, - query_count: integer(value, "query_count")?, - attributed_seconds: decimal(value, "attributed_seconds")?, - bytes_scanned: integer(value, "bytes_scanned")?, - estimated_compute_credits: decimal(value, "estimated_compute_credits")?, - estimated_usd: decimal(value, "estimated_usd")?, - }) -} - -pub(crate) fn summary(value: &Bound<'_, PyAny>) -> PyResult { - let mut resources: Vec = Vec::new(); - for item in native_field(value, "resources")?.try_iter()? { - resources.push(resource(&item?)?); - } - let mut limitations: Vec = Vec::new(); - for item in native_field(value, "limitations")?.try_iter()? { - limitations.push(item?.extract()?); - } - Ok(CostSummary { - status: native_field(value, "status")?.getattr("value")?.extract()?, - usd_per_credit: decimal(value, "usd_per_credit")?, - rate_source: text(value, "rate_source")?, - resources, - query_count: integer(value, "query_count")?, - attributed_seconds: decimal(value, "attributed_seconds")?, - bytes_scanned: integer(value, "bytes_scanned")?, - estimated_compute_credits: decimal(value, "estimated_compute_credits")?, - estimated_usd: decimal(value, "estimated_usd")?, - expected_statement_count: integer(value, "expected_statement_count")?, - observed_statement_count: integer(value, "observed_statement_count")?, - missing_statement_count: integer(value, "missing_statement_count")?, - skipped_statement_count: integer(value, "skipped_statement_count")?, - source: text(value, "source")?, - method: text(value, "method")?, - limitations, - message: optional_text(value, "message")?, - }) -} - -/// A `CostRunRecord` as a plain run. -pub(crate) fn cost_run(record: &Bound<'_, PyAny>) -> PyResult { - Ok(CostRun { - run_id: text(record, "run_id")?, - adapter_name: text(record, "adapter_name")?, - target_name: optional_text(record, "target_name")?, - build_status: text(record, "build_status")?, - started_at: timestamp(record, "started_at")?, - completed_at: timestamp(record, "completed_at")?, - cost: summary(&native_field(record, "cost")?)?, - version: integer(record, "version")?, - had_executable_work: native_field(record, "had_executable_work")?.extract()?, - }) -} - -/// Every record of an iterable as a plain run. -pub(crate) fn cost_runs(records: &Bound<'_, PyAny>) -> PyResult> { - let mut runs: Vec = Vec::new(); - for record in records.try_iter()? { - runs.push(cost_run(&record?)?); - } - Ok(runs) -} - -/// Python constructors of the record fields. -struct Builders<'py> { - decimal: Bound<'py, PyAny>, - timestamp: Bound<'py, PyAny>, -} - -impl<'py> Builders<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - decimal: imported(py, "decimal", "Decimal")?, - timestamp: imported(py, "datetime", "datetime")?.getattr("fromisoformat")?, - }) - } - - fn decimal(&self, text: &str) -> PyResult> { - self.decimal.call1((text,)) - } -} - -fn resource_object<'py>( - builders: &Builders<'py>, - resource: &CostResource, -) -> PyResult> { - let py: Python<'py> = builders.decimal.py(); - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("resource_type", &resource.resource_type)?; - fields.set_item("resource_name", &resource.resource_name)?; - fields.set_item("warehouse_name", &resource.warehouse_name)?; - fields.set_item("warehouse_size", &resource.warehouse_size)?; - fields.set_item("warehouse_type", &resource.warehouse_type)?; - fields.set_item("cluster_number", resource.cluster_number)?; - fields.set_item("query_count", resource.query_count)?; - fields.set_item( - "attributed_seconds", - builders.decimal(&resource.attributed_seconds)?, - )?; - fields.set_item("bytes_scanned", resource.bytes_scanned)?; - fields.set_item( - "estimated_compute_credits", - builders.decimal(&resource.estimated_compute_credits)?, - )?; - fields.set_item("estimated_usd", builders.decimal(&resource.estimated_usd)?)?; - new_record(py, RecordId::ResourceCost, (), Some(&fields)) -} - -fn summary_object<'py>( - builders: &Builders<'py>, - cost: &CostSummary, -) -> PyResult> { - let py: Python<'py> = builders.decimal.py(); - let mut resources: Vec> = Vec::with_capacity(cost.resources.len()); - for resource in &cost.resources { - resources.push(resource_object(builders, resource)?); - } - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item( - "status", - host_entry(py, ExecutorHost::CostStatus)?.call1((cost.status.as_str(),))?, - )?; - fields.set_item("usd_per_credit", builders.decimal(&cost.usd_per_credit)?)?; - fields.set_item("rate_source", &cost.rate_source)?; - fields.set_item("resources", PyTuple::new(py, resources)?)?; - fields.set_item("query_count", cost.query_count)?; - fields.set_item( - "attributed_seconds", - builders.decimal(&cost.attributed_seconds)?, - )?; - fields.set_item("bytes_scanned", cost.bytes_scanned)?; - fields.set_item( - "estimated_compute_credits", - builders.decimal(&cost.estimated_compute_credits)?, - )?; - fields.set_item("estimated_usd", builders.decimal(&cost.estimated_usd)?)?; - fields.set_item("expected_statement_count", cost.expected_statement_count)?; - fields.set_item("observed_statement_count", cost.observed_statement_count)?; - fields.set_item("missing_statement_count", cost.missing_statement_count)?; - fields.set_item("skipped_statement_count", cost.skipped_statement_count)?; - fields.set_item("source", &cost.source)?; - fields.set_item("method", &cost.method)?; - fields.set_item("limitations", PyTuple::new(py, &cost.limitations)?)?; - fields.set_item("message", &cost.message)?; - host_entry(py, ExecutorHost::RunCostSummary)?.call((), Some(&fields)) -} - -/// A plain run as the `CostRunRecord` it was read from. -pub(crate) fn record_object<'py>(py: Python<'py>, run: &CostRun) -> PyResult> { - let builders: Builders<'py> = Builders::new(py)?; - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("version", run.version)?; - fields.set_item("run_id", &run.run_id)?; - fields.set_item("adapter_name", &run.adapter_name)?; - fields.set_item("target_name", &run.target_name)?; - fields.set_item("build_status", &run.build_status)?; - fields.set_item( - "started_at", - builders.timestamp.call1((run.started_at.as_str(),))?, - )?; - fields.set_item( - "completed_at", - builders.timestamp.call1((run.completed_at.as_str(),))?, - )?; - fields.set_item("cost", summary_object(&builders, &run.cost)?)?; - fields.set_item("had_executable_work", run.had_executable_work)?; - new_record(py, RecordId::CostRunRecord, (), Some(&fields)) -} - -/// Records of several plain runs, as a tuple. -pub(crate) fn record_tuple<'py>( - py: Python<'py>, - runs: &[CostRun], -) -> PyResult> { - let mut records: Vec> = Vec::with_capacity(runs.len()); - for run in runs { - records.push(record_object(py, run)?); - } - PyTuple::new(py, records) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs index ecc1512232..a45f150a11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs @@ -9,7 +9,7 @@ use sqlbuild_cli::cost_store::main::read_cost_run::read_cost_run; use sqlbuild_cli::cost_store::main::write_cost_run::write_cost_run; use sqlbuild_core::cost_report::models::CostRun; -use crate::bindings::_helpers::cost_report::records::artifact_error; +use sqlbuild_python_front::resource_costs::main::artifact_error::artifact_error; fn failure(py: Python<'_>, error: CostStoreError) -> PyErr { match error { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 7a88a487a1..2bf84302ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -29,6 +29,7 @@ use crate::bindings::_helpers::cli::dispatch::warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::WarehouseInputs; use crate::bindings::_helpers::command_runtime::warehouses::configured_warehouses; use crate::bindings::_helpers::compile_driver::rules_at_load::RulesAtLoadScope; +use crate::bindings::_helpers::cost_report::deferred::collect_deferred_cost_after_finish; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; use crate::bindings::_helpers::location_command::constants::CHANGED_SETTING; @@ -396,7 +397,9 @@ pub(super) fn execute_worker<'py>( let outcome = if recall.is_some() { worker.outcome(py, code)? } else { - worker.finish(py, code)? + let outcome = worker.finish(py, code)?; + collect_deferred_cost_after_finish(py, &worker); + outcome }; result.map(|code| { if matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index a0c0027b74..b39e4a82a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; use serde_json::{Value, json}; use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGERPRINT_COLUMNS}; use sqlbuild_config::command_connection::constants::{ - DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, + CONNECTION_WAREHOUSE_KEY, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, @@ -42,8 +42,10 @@ use crate::bindings::_helpers::query_command::project::project_adapter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::constants::HISTORY_STREAM_PAGES; +use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; @@ -92,6 +94,22 @@ pub(crate) struct HistoryStream<'py> { since: Option>, } +/// Say which warehouse a sync session runs on, so a deployment sees it is not the largest one. +fn report_sync_warehouse(py: Python<'_>, connection_config: &Bound<'_, PyAny>) -> PyResult<()> { + let warehouse: Bound<'_, PyAny> = mapping_get(connection_config, CONNECTION_WAREHOUSE_KEY)?; + if warehouse.is_none() { + return Ok(()); + } + write_console( + py, + "stderr", + &format!( + "Warehouse sync session opened on warehouse {}.\n", + warehouse.str()? + ), + ) +} + pub(crate) fn open_history_session<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, @@ -111,6 +129,7 @@ pub(crate) fn open_history_session<'py>( (&py.None().into_bound(py), true), true, )?; + report_sync_warehouse(py, &connection_config)?; let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; let local_path_key = LOCAL_DATABASE_PATH_KEYS .iter() diff --git a/crates/sqlbuild-state/src/resource_costs/constants.rs b/crates/sqlbuild-state/src/resource_costs/constants.rs index 4ce0fe26be..957ca4838b 100644 --- a/crates/sqlbuild-state/src/resource_costs/constants.rs +++ b/crates/sqlbuild-state/src/resource_costs/constants.rs @@ -59,6 +59,8 @@ pub const LATEST_RUN_COST_COLUMNS: [&str; 11] = [ pub const RESOURCE_COST_TAG: &str = "resource_cost"; /// Statuses a later build re-reads while the warehouse is awake. pub const REFRESHABLE_COST_STATUSES: [&str; 3] = ["pending", "partial", "collection_failed"]; +/// Status re-read only while a statement was not visible yet or still running in query history. +pub const PARTIAL_STATUS: &str = "partial"; /// Status whose re-reads are bounded by `MAX_FAILED_COLLECTION_ATTEMPTS`. pub const COLLECTION_FAILED_STATUS: &str = "collection_failed"; /// Collections of a run, the first included, after which a failed collection is not retried. @@ -67,5 +69,7 @@ pub const MAX_FAILED_COLLECTION_ATTEMPTS: i64 = 3; pub const REFRESH_WINDOW_DAYS: i64 = 6; /// Most runs one build re-reads, newest first. pub const MAX_REFRESHED_RUNS: usize = 3; +/// Seconds after a run ends before a later build re-reads it, once its history has settled. +pub const REFRESH_SETTLE_SECONDS: i64 = 120; /// Seconds in one day, for the refresh window. pub const SECONDS_PER_DAY: i64 = 86_400; diff --git a/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs index 0396acc699..f4d7b18cf5 100644 --- a/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs +++ b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs @@ -1,12 +1,24 @@ //! Which runs a build re-reads while the warehouse is awake. +use sqlbuild_core::cost_command::constants::CHANGEABLE_LIMITATIONS; +use sqlbuild_core::cost_command::models::CostCoverage; + use crate::resource_costs::constants::{ - COLLECTION_FAILED_STATUS, MAX_FAILED_COLLECTION_ATTEMPTS, MAX_REFRESHED_RUNS, - REFRESH_WINDOW_DAYS, REFRESHABLE_COST_STATUSES, SECONDS_PER_DAY, + COLLECTION_FAILED_STATUS, MAX_FAILED_COLLECTION_ATTEMPTS, MAX_REFRESHED_RUNS, PARTIAL_STATUS, + REFRESH_SETTLE_SECONDS, REFRESH_WINDOW_DAYS, REFRESHABLE_COST_STATUSES, SECONDS_PER_DAY, }; use crate::resource_costs::models::RunCostAttempt; -/// Newest refreshable runs in the history window, skipping exhausted failures; `now` is epoch seconds. +/// Whether a partial run can still change: a statement unseen, running or lacking settling metadata. +fn partial_can_change(coverage: &CostCoverage) -> bool { + coverage.observed_statement_count < coverage.expected_statement_count + || coverage + .limitations + .iter() + .any(|limitation| CHANGEABLE_LIMITATIONS.contains(&limitation.as_str())) +} + +/// Newest settled runs a re-read can still change, skipping exhausted failures; `now` is epoch seconds. #[must_use] pub fn refresh_candidates(latest: &[RunCostAttempt], now: i64) -> Vec { let oldest: i64 = now - REFRESH_WINDOW_DAYS * SECONDS_PER_DAY; @@ -14,6 +26,8 @@ pub fn refresh_candidates(latest: &[RunCostAttempt], now: i64) -> Vec= oldest) + .filter(|run| now - run.completed_epoch >= REFRESH_SETTLE_SECONDS) + .filter(|run| run.coverage.status != PARTIAL_STATUS || partial_can_change(&run.coverage)) .filter(|run| { run.coverage.status != COLLECTION_FAILED_STATUS || run.attempt < MAX_FAILED_COLLECTION_ATTEMPTS diff --git a/crates/sqlbuild-state/src/resource_costs/models.rs b/crates/sqlbuild-state/src/resource_costs/models.rs index f1d400938d..2cc0f317ec 100644 --- a/crates/sqlbuild-state/src/resource_costs/models.rs +++ b/crates/sqlbuild-state/src/resource_costs/models.rs @@ -16,6 +16,8 @@ pub struct RunCostAttempt { pub run_completed_at: String, /// Seconds since the epoch of the run's start. pub started_epoch: i64, + /// Seconds since the epoch of the run's end. + pub completed_epoch: i64, } /// What identifies one observation: the run, the collection attempt, the resource and its warehouse. diff --git a/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs b/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs index 49237b71ee..0b53533fe8 100644 --- a/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs +++ b/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs @@ -24,9 +24,18 @@ pub(crate) fn run(run_id: &str, attempt: i64, status: &str, age: i64) -> RunCost run_started_at: String::new(), run_completed_at: String::new(), started_epoch: NOW - age, + completed_epoch: NOW - age + 60, } } +/// A partial run that saw every statement, with `limitations`. +pub(crate) fn fully_observed(run_id: &str, limitations: &[&str]) -> RunCostAttempt { + let mut attempt: RunCostAttempt = run(run_id, 1, "partial", 3_600); + attempt.coverage.observed_statement_count = attempt.coverage.expected_statement_count; + attempt.coverage.limitations = limitations.iter().map(|text| (*text).to_owned()).collect(); + attempt +} + pub(crate) fn observation_id(attempt: i64, cluster_number: Option) -> String { resource_cost_observation_id(&ResourceCostKey { run_id: "run", diff --git a/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs index fb9ef8b77c..19de16e4d3 100644 --- a/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs +++ b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs @@ -1,5 +1,9 @@ use crate::resource_costs::main::refresh_candidates::refresh_candidates; -use crate::resource_costs::tests::helpers::{NOW, observation_id, run}; +use sqlbuild_core::cost_command::constants::{ + INCOMPLETE_HISTORY_LIMITATION, MISSING_METADATA_LIMITATION, +}; + +use crate::resource_costs::tests::helpers::{NOW, fully_observed, observation_id, run}; use crate::resource_costs::tests::test_types::{ObservationIdTestCase, RefreshTestCase}; const HOUR: i64 = 3_600; @@ -18,6 +22,30 @@ fn given_latest_run_rows_when_choosing_refreshes_then_only_bounded_recent_incomp ], expected_runs: vec!["pending", "partial"], }, + RefreshTestCase { + description: "a partial run that saw every statement is final unless history can still fill in", + latest: vec![ + fully_observed( + "permanent", + &[ + "Concurrency sharing includes only queries visible to the executing Snowflake role.", + "Snowflake query-history result limit was reached.", + ], + ), + fully_observed("still_running", &[INCOMPLETE_HISTORY_LIMITATION]), + fully_observed("recent_metadata", &[MISSING_METADATA_LIMITATION]), + run("unseen_statement", 1, "partial", 2 * HOUR), + ], + expected_runs: vec!["recent_metadata", "still_running", "unseen_statement"], + }, + RefreshTestCase { + description: "a run that ended under two minutes ago has not settled yet", + latest: vec![ + run("just_ended", 1, "pending", 150), + run("settled", 1, "pending", 180), + ], + expected_runs: vec!["settled"], + }, RefreshTestCase { description: "a failed collection is retried until its third attempt, not after", latest: vec![ diff --git a/fensu.toml b/fensu.toml index 9e9faafec2..4c7f00eae3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2721,7 +2721,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs=9", @@ -3036,7 +3035,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=2", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md index fff88cb037..5598592a37 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md @@ -71,6 +71,7 @@ Configuration: schema: DEV [OK resolved] build warehouse: BUILD_WH [OK sqlbuild_project.toml [targets.dev.warehouses] build] query warehouse: TRANSFORM_WH [OK connection warehouse] + sync warehouse: TRANSFORM_WH [OK connection warehouse] Providers: providers: none discovered [OK none discovered] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 08b57421cf..dcedd89b3c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -84,6 +84,7 @@ query = "ADHOC_WH" |-------|----------| | `build` | `audit`, `build`, `check`, `clone`, `dbt` (SQLBuild's own work), `load`, `plan`, `seed` | | `query` | `contract`, `cost`, `debug`, `diff`, `freshness`, `janitor`, `query`, `scenario`, `test` | +| `sync` | `location`: a code location's warehouse syncs, which read metadata and SQLBuild's small state tables | Each command uses the first of these that is set: @@ -92,6 +93,20 @@ Each command uses the first of these that is set: 3. The connection's `warehouse` 4. The Snowflake user's default warehouse +A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the +connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read +metadata. Only when the connection sets no `warehouse` does a sync use the `build` warehouse. Set +`sync` when the syncs need their own warehouse: + +```toml +[targets.dev.warehouses] +build = "BUILD_WH" +query = "ADHOC_WH" +sync = "TINY_WH" +``` + +Runs that a location executes, such as `sqb build`, use their own command's group. + Every connection a command opens uses the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index ed51b47f92..63c1f46108 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -185,7 +185,7 @@ A target is a named build context - the database and schema you build into, plus | `defer_sources_to` | Target name to read managed source data from (see [Loaders](python-nodes/loaders.md#source-deferral)) | | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny`. See [Missing migration origins](models/migrations.md#missing-migration-origins) | -| `warehouses` | Snowflake warehouse per command group, `build` and `query`. See [Command-group warehouses](adapters/snowflake.md#command-group-warehouses) | +| `warehouses` | Snowflake warehouse per command group: `build`, `query` and `sync` (a location's warehouse syncs). See [Command-group warehouses](adapters/snowflake.md#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index de71fe3845..54daf3725c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -11742,6 +11742,7 @@ class WarehouseGroup(StrEnum): BUILD = "build" QUERY = "query" + SYNC = "sync" class SourceWriteStrategy(StrEnum): APPEND = "append" @@ -11954,10 +11955,12 @@ class TargetWarehousesConfig(NativeRecord): build: str | None query: str | None + sync: str | None def __init__( self, build: str | None = ..., query: str | None = ..., + sync: str | None = ..., ) -> None: ... class TargetConfig(NativeRecord): diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index bf38d9db07..3f09263a98 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -77,6 +77,43 @@ expected_warehouse="BUILD_WH", expected_source="sqlbuild_project.toml [targets.dev.warehouses] build", ), + CommandWarehouseResolutionTestCase( + description="a location syncs on the query warehouse, not the connection's", + command="location", + project_warehouses=_PROJECT_GROUPS, + expected_warehouse="ADHOC_WH", + expected_source="sqlbuild_project.toml [targets.dev.warehouses] query", + ), + CommandWarehouseResolutionTestCase( + description="a location syncs on the sync warehouse when the target sets one", + command="location", + project_warehouses=TargetWarehousesConfig( + build="BUILD_WH", query="ADHOC_WH", sync="SYNC_WH" + ), + expected_warehouse="SYNC_WH", + expected_source="sqlbuild_project.toml [targets.dev.warehouses] sync", + ), + CommandWarehouseResolutionTestCase( + description="a location with only a build warehouse syncs on the connection's, not the build one", + command="location", + project_warehouses=TargetWarehousesConfig(build="XL_WH"), + expected_warehouse="ANALYTICS_WH", + expected_source="connection warehouse", + ), + CommandWarehouseResolutionTestCase( + description="a location with only a build warehouse and no connection warehouse syncs on the build one", + command="location", + project_warehouses=TargetWarehousesConfig(build="XL_WH"), + connection={"account": "example-account"}, + expected_warehouse="XL_WH", + expected_source="sqlbuild_project.toml [targets.dev.warehouses] build", + ), + CommandWarehouseResolutionTestCase( + description="a location whose target sets no warehouse keeps the connection's", + command="location", + expected_warehouse="ANALYTICS_WH", + expected_source="connection warehouse", + ), CommandWarehouseResolutionTestCase( description="cli warehouse overrides the group", command="build", @@ -361,6 +398,11 @@ def test_given_command_without_group_when_parsing_warehouse_flag_then_it_is_reje command=CliCommand.BUILD, expected_group=WarehouseGroup.BUILD, ), + CommandWarehouseGroupTestCase( + description="a location's syncs read metadata and small state tables", + command=CliCommand.LOCATION, + expected_group=WarehouseGroup.SYNC, + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index 413b040910..9b62cb5992 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -63,6 +63,11 @@ class DuckDbPlusAdapter(DuckDbAdapter): + '\n\n[targets.dev.warehouses]\nbuild = "BUILD_WH"\nquery = "ADHOC_WH"\n', expected_warehouses=TargetWarehousesConfig(build="BUILD_WH", query="ADHOC_WH"), ), + TargetWarehousesConfigTestCase( + description="project target sets a sync warehouse for its location", + contents=_SNOWFLAKE_PROJECT + '\n\n[targets.dev.warehouses]\nsync = "SYNC_WH"\n', + expected_warehouses=TargetWarehousesConfig(sync="SYNC_WH"), + ), TargetWarehousesConfigTestCase( description="target without warehouses keeps both groups unset", contents=_SNOWFLAKE_PROJECT, diff --git a/tests/unit/src/sqlbuild/cost/_test_types.py b/tests/unit/src/sqlbuild/cost/_test_types.py index 145f865885..8ddcec4784 100644 --- a/tests/unit/src/sqlbuild/cost/_test_types.py +++ b/tests/unit/src/sqlbuild/cost/_test_types.py @@ -46,6 +46,7 @@ class SnowflakeObservationTestCase: expected_limitation_fragment: str = "" query_type: str = "SELECT" query_text: str = "SELECT * FROM orders" + observed_after_seconds: int = 0 @dataclass(frozen=True) @@ -147,3 +148,10 @@ class SnowflakeCostBudgetTestCase: seconds_per_read: float expected_status: CostStatus expected_message_fragment: str = "" + + +@dataclass(frozen=True) +class HistoryWindowTestCase: + description: str + completed_minutes_ago: int + expected_bounded: bool diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake.py b/tests/unit/src/sqlbuild/cost/test_snowflake.py index fabafcbfca..b2fe35676b 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake.py @@ -21,6 +21,7 @@ from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import ( CollectSnowflakeCostTestCase, + HistoryWindowTestCase, ParseQueryTagTestCase, RenderQueryHistorySqlTestCase, SnowflakeClockSkewTestCase, @@ -227,7 +228,7 @@ def test_given_exact_create_hook_when_collecting_then_hook_query_text_is_not_req def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not_saturated( test_case: SnowflakeSaturationTestCase, ) -> None: - origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) + origin: datetime = datetime.now(UTC) - timedelta(minutes=1) row: tuple[Any, ...] = ( "q-open", "", @@ -564,6 +565,37 @@ def test_given_ledger_persistence_failure_when_collecting_then_summary_is_partia expected_query_count=0, expected_limitation_fragment="missing Snowflake warehouse metadata", ), + SnowflakeObservationTestCase( + description="a settled query without warehouse size used no warehouse", + warehouse_name="DEV_WH", + warehouse_size=None, + execution_ms=2, + execution_status="SUCCESS", + expected_status=CostStatus.COMPLETE, + expected_query_count=0, + observed_after_seconds=121, + ), + SnowflakeObservationTestCase( + description="a settled query with zero execution used no warehouse", + warehouse_name="DEV_WH", + warehouse_size="X-Small", + execution_ms=0, + execution_status="SUCCESS", + expected_status=CostStatus.COMPLETE, + expected_query_count=0, + observed_after_seconds=121, + ), + SnowflakeObservationTestCase( + description="a query read just before it settles still lacks metadata", + warehouse_name="DEV_WH", + warehouse_size=None, + execution_ms=1000, + execution_status="SUCCESS", + expected_status=CostStatus.PARTIAL, + expected_query_count=0, + expected_limitation_fragment="missing Snowflake warehouse metadata", + observed_after_seconds=119, + ), SnowflakeObservationTestCase( description="cancelled cost-bearing statement remains attributable", warehouse_name="DEV_WH", @@ -642,7 +674,7 @@ def test_given_special_history_row_when_collecting_then_coverage_is_explicit( test_case.execution_status, test_case.query_type, test_case.query_text, - origin + timedelta(seconds=1), + origin + timedelta(seconds=1 + test_case.observed_after_seconds), ), ) @@ -741,15 +773,16 @@ def test_given_metadata_only_create_when_warehouse_size_is_absent_then_cost_is_c "test_case", [ CollectSnowflakeCostTestCase( - description="delayed insert metadata is retried", - expected_status=CostStatus.COMPLETE, - expected_query_count=1, - expected_resource_name="countries", + description="delayed insert metadata is left for a later build's re-read", + expected_status=CostStatus.PARTIAL, + expected_query_count=0, + expected_resource_name="", + expected_limitation_fragment="missing Snowflake warehouse metadata", ) ], ids=lambda case: case.description, ) -def test_given_compute_query_with_delayed_metadata_when_collecting_then_history_is_retried( +def test_given_compute_query_with_delayed_metadata_when_collecting_then_it_is_not_retried_at_run_end( test_case: CollectSnowflakeCostTestCase, tmp_path: Path, ) -> None: @@ -808,10 +841,10 @@ def test_given_compute_query_with_delayed_metadata_when_collecting_then_history_ retry_delay_seconds=0, ) - assert connection.call_count == 2 + assert connection.call_count == 1 assert summary.status == test_case.expected_status assert summary.query_count == test_case.expected_query_count - assert summary.resources[0].resource_name == test_case.expected_resource_name + assert test_case.expected_limitation_fragment in " ".join(summary.limitations) @pytest.mark.parametrize( @@ -1168,7 +1201,7 @@ def test_given_saturated_open_history_when_bounded_fallback_completes_then_runni test_case: CollectSnowflakeCostTestCase, tmp_path: Path, ) -> None: - origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) + origin: datetime = datetime.now(UTC) - timedelta(minutes=1) ledger_path: Path = tmp_path / "statements.jsonl" ledger_path.write_text( json.dumps( @@ -1295,3 +1328,37 @@ def test_given_run_older_than_retention_when_history_is_missing_then_status_is_u assert summary.status == test_case.expected_status assert summary.query_count == test_case.expected_query_count assert "no longer available" in (summary.message or "") + + +@pytest.mark.parametrize( + "test_case", + [ + HistoryWindowTestCase( + description="a run that just finished reads open-ended history for running queries", + completed_minutes_ago=0, + expected_bounded=False, + ), + HistoryWindowTestCase( + description="an older run reads only its own window, not the hours since", + completed_minutes_ago=30, + expected_bounded=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_run_age_when_reading_history_then_older_runs_read_a_bounded_window( + test_case: HistoryWindowTestCase, +) -> None: + completed_at: datetime = datetime.now(UTC) - timedelta(minutes=test_case.completed_minutes_ago) + connection: _Connection = _Connection(()) + + snowflake_query_observations( + connection=connection, + started_at=completed_at - timedelta(minutes=1), + completed_at=completed_at, + expected_query_ids=frozenset(), + classification_query_ids=frozenset(), + deadline=float("inf"), + ) + + assert ("END_TIME_RANGE_END" in connection.sql) is test_case.expected_bounded diff --git a/website/src/content/docs/docs/cli/debug.mdx b/website/src/content/docs/docs/cli/debug.mdx index d726fc0b05..bb9022fabc 100644 --- a/website/src/content/docs/docs/cli/debug.mdx +++ b/website/src/content/docs/docs/cli/debug.mdx @@ -70,6 +70,7 @@ Configuration: schema: DEV [OK resolved] build warehouse: BUILD_WH [OK sqlbuild_project.toml [targets.dev.warehouses] build] query warehouse: TRANSFORM_WH [OK connection warehouse] + sync warehouse: TRANSFORM_WH [OK connection warehouse] Providers: providers: none discovered [OK none discovered] diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index bb910975cb..c0c1319e5b 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -69,6 +69,7 @@ query = "ADHOC_WH" |-------|----------| | `build` | `audit`, `build`, `check`, `clone`, `dbt` (SQLBuild's own work), `load`, `plan`, `seed` | | `query` | `contract`, `cost`, `debug`, `diff`, `freshness`, `janitor`, `query`, `scenario`, `test` | +| `sync` | `location`: a code location's warehouse syncs, which read metadata and SQLBuild's small state tables | Each command uses the first of these that is set: @@ -77,6 +78,20 @@ Each command uses the first of these that is set: 3. The connection's `warehouse` 4. The Snowflake user's default warehouse +A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the +connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read +metadata. Only when the connection sets no `warehouse` does a sync use the `build` warehouse. Set +`sync` when the syncs need their own warehouse: + +```toml +[targets.dev.warehouses] +build = "BUILD_WH" +query = "ADHOC_WH" +sync = "TINY_WH" +``` + +Runs that a location executes, such as `sqb build`, use their own command's group. + Every connection a command opens uses the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 2da808b91d..908c9fd342 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -159,7 +159,7 @@ A target is a named build context - the database and schema you build into, plus | `defer_sources_to` | Target name to read managed source data from (see [Loaders](/docs/concepts/python-nodes/loaders#source-deferral)) | | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny`. See [Missing migration origins](/docs/concepts/models/migrations#missing-migration-origins) | -| `warehouses` | Snowflake warehouse per command group, `build` and `query`. See [Command-group warehouses](/docs/concepts/adapters/snowflake#command-group-warehouses) | +| `warehouses` | Snowflake warehouse per command group: `build`, `query` and `sync` (a location's warehouse syncs). See [Command-group warehouses](/docs/concepts/adapters/snowflake#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) |