From 5d6f8ebdd68d1d1f6280e1449b0fb7cd5f062446 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:58:23 +0000 Subject: [PATCH 1/3] wip: record estimated query cost per run, resource and warehouse in the state schema Not yet complete: no UI, server reads, docs or integration tests yet. Rust unit tests for the refresh rules pass; clippy is clean. --- .../src/janitor_planning/constants.rs | 2 +- crates/sqlbuild-core/src/constants.rs | 5 +- .../_helpers/resources/warehouse_metadata.rs | 2 + .../src/store/schema.sql | 21 ++ .../main/ingest/ingest_resource_costs.rs | 71 ++++ .../src/warehouse_metadata/main/ingest/mod.rs | 1 + .../src/warehouse_metadata/models.rs | 45 ++- .../_helpers/build_command/completion.rs | 10 + .../src/bindings/_helpers/cli/registration.rs | 1 + .../compiled_project/snapshot_execution.rs | 5 + .../_helpers/cost_report/degradation.rs | 37 +++ .../src/bindings/_helpers/cost_report/mod.rs | 1 + .../_helpers/cost_report/publication.rs | 171 ++++++++++ .../bindings/_helpers/cost_report/records.rs | 2 +- .../_helpers/janitor/janitor_facts.rs | 11 +- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/resource_costs_state.rs | 304 ++++++++++++++++++ .../native_host/warehouse_metadata.rs | 31 +- .../bindings/_helpers/records/constants.rs | 2 + crates/sqlbuild-state/src/lib.rs | 1 + .../src/resource_costs/constants.rs | 70 ++++ .../main/latest_run_costs_sql.rs | 20 ++ .../src/resource_costs/main/mod.rs | 4 + .../main/prune_resource_costs_sql.rs | 14 + .../resource_costs/main/refresh_candidates.rs | 34 ++ .../main/resource_cost_observation_id.rs | 28 ++ .../sqlbuild-state/src/resource_costs/mod.rs | 6 + .../src/resource_costs/models.rs | 32 ++ .../src/resource_costs/tests.rs | 4 + .../src/resource_costs/tests/test_refresh.rs | 122 +++++++ .../src/resource_costs/tests/test_types.rs | 14 + src/sqlbuild/_native.pyi | 18 ++ .../contract/classes/resource_costs.py | 37 +++ .../snowflake/classes/snowflake_adapter.py | 11 + 34 files changed, 1127 insertions(+), 11 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/constants.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/main/mod.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/main/prune_resource_costs_sql.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/main/resource_cost_observation_id.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/mod.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/models.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/tests.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs create mode 100644 crates/sqlbuild-state/src/resource_costs/tests/test_types.rs create mode 100644 src/sqlbuild/adapter/contract/classes/resource_costs.py diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 533e9fefe6..afbf2f8d88 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -3,7 +3,7 @@ use sqlbuild_core::constants::STATE_TABLE_NAMES; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 11] = STATE_TABLE_NAMES; +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 12] = STATE_TABLE_NAMES; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; /// Separator between archive name segments. diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 1dc5865b99..63fc1820a8 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -40,8 +40,10 @@ pub const AUDIT_RESULTS_TABLE_NAME: &str = "_sqlbuild_audit_results"; pub const JANITOR_EVENTS_TABLE_NAME: &str = "_sqlbuild_janitor_events"; /// State table of display-only observations, which plan previews read and planning never does. pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; +/// State table of estimated query cost per run, resource and warehouse; display only. +pub const RESOURCE_COSTS_TABLE_NAME: &str = "_sqlbuild_resource_costs"; /// Every state table SQLBuild writes in a target's state schema. -pub const STATE_TABLE_NAMES: [&str; 11] = [ +pub const STATE_TABLE_NAMES: [&str; 12] = [ FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, @@ -53,6 +55,7 @@ pub const STATE_TABLE_NAMES: [&str; 11] = [ COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, OBSERVATIONS_TABLE_NAME, + RESOURCE_COSTS_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs index a839f6cd02..bc4a044b1e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/warehouse_metadata.rs @@ -19,6 +19,7 @@ use crate::warehouse_metadata::main::ingest::ingest_migrations::ingest_migration use crate::warehouse_metadata::main::ingest::ingest_node_results::ingest_node_results; use crate::warehouse_metadata::main::ingest::ingest_observations::ingest_observations; use crate::warehouse_metadata::main::ingest::ingest_old_name_views::ingest_old_name_views; +use crate::warehouse_metadata::main::ingest::ingest_resource_costs::ingest_resource_costs; use crate::warehouse_metadata::main::ingest::ingest_source_freshness::ingest_source_freshness; use crate::warehouse_metadata::main::ingest::ingest_statistics::ingest_statistics; use crate::warehouse_metadata::main::location_reads::read_preview_copy::read_preview_copy; @@ -60,6 +61,7 @@ pub(crate) async fn report( WarehouseHistoryKind::NodeResults => ingest_node_results(&store, scope, &batch).await, WarehouseHistoryKind::Consumption => ingest_consumption(&store, scope, &batch).await, WarehouseHistoryKind::Observations => ingest_observations(&store, scope, &batch).await, + WarehouseHistoryKind::ResourceCosts => ingest_resource_costs(&store, scope, &batch).await, WarehouseHistoryKind::Catalog => ingest_catalog(&store, scope, &batch).await, WarehouseHistoryKind::Statistics => ingest_statistics(&store, scope, &batch).await, }?; diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index fafa971546..cd25e62407 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -518,6 +518,27 @@ CREATE TABLE sqb_warehouse_observations ( ); CREATE INDEX sqb_observations_namespace ON sqb_warehouse_observations (location_id,target,state_database,state_schema,observed_at,event_id); CREATE INDEX sqb_observations_latest ON sqb_warehouse_observations (location_id,target,kind,node,observed_at,event_id); +CREATE TABLE sqb_warehouse_resource_costs ( + event_id {uuid} PRIMARY KEY, + location_id {uuid} NOT NULL REFERENCES sqb_locations(location_id), + target TEXT NOT NULL, + observed_at {time} NOT NULL, + state_database TEXT, + state_schema TEXT NOT NULL, + run_id TEXT NOT NULL, + orchestration_run_id TEXT, + attempt BIGINT NOT NULL, + resource_type TEXT NOT NULL, + resource_name TEXT NOT NULL, + run_started_at {time} NOT NULL, + record_json {json} NOT NULL, + received_at {time} NOT NULL, + scan_id {uuid} +); +CREATE INDEX sqb_resource_costs_namespace ON sqb_warehouse_resource_costs (location_id,target,state_database,state_schema,observed_at,event_id); +CREATE INDEX sqb_resource_costs_run ON sqb_warehouse_resource_costs (location_id,target,run_id,attempt); +CREATE INDEX sqb_resource_costs_orchestration_run ON sqb_warehouse_resource_costs (orchestration_run_id); +CREATE INDEX sqb_resource_costs_started ON sqb_warehouse_resource_costs (location_id,target,run_started_at); CREATE INDEX sqb_freshness_source_latest ON sqb_warehouse_freshness ( location_id, target, (record_json ->> 'source_name'), (record_json ->> 'target_database'), (record_json ->> 'target_schema'), diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs new file mode 100644 index 0000000000..1871724315 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs @@ -0,0 +1,71 @@ +//! Mirror display-only estimated query cost observations; each refresh is another observation. + +use serde_json::json; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_metadata::_helpers::events::ingest_events; +use crate::warehouse_metadata::_helpers::ingestion::batch_records; +use crate::warehouse_metadata::models::{ + MetadataBatch, StateEventImport, WarehouseResourceCostRecord, WarehouseStateLocation, + WarehouseSyncResponse, +}; + +pub async fn ingest_resource_costs( + store: &ControlStore, + scope: (Uuid, &str), + batch: &MetadataBatch, +) -> Result { + let records: Vec = batch_records(batch)?; + let invalid_identity: bool = records.iter().any(|record| { + record.attempt < 1 + || [ + &record.observation_id, + &record.state_schema, + &record.run_id, + &record.resource_type, + &record.resource_name, + ] + .iter() + .any(|value| value.trim().is_empty()) + }); + ingest_events( + store, + scope, + batch, + ( + records + .iter() + .map(|record| StateEventImport { + source_id: record.observation_id.clone(), + location: WarehouseStateLocation { + database: record.state_database.clone(), + schema: record.state_schema.clone(), + }, + observed_at: record.observed_at, + columns: vec![ + ("run_id", Parameter::Text(record.run_id.clone())), + ( + "orchestration_run_id", + Parameter::OptionalText(record.orchestration_run_id.clone()), + ), + ("attempt", Parameter::Integer(record.attempt)), + ( + "resource_type", + Parameter::Text(record.resource_type.clone()), + ), + ( + "resource_name", + Parameter::Text(record.resource_name.clone()), + ), + ("run_started_at", Parameter::Time(record.run_started_at)), + ], + record_json: json!(record), + }) + .collect(), + invalid_identity, + ), + ) + .await +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs index e54d3083df..c51faaae90 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/mod.rs @@ -9,6 +9,7 @@ pub mod ingest_migrations; pub mod ingest_node_results; pub mod ingest_observations; pub mod ingest_old_name_views; +pub mod ingest_resource_costs; pub mod ingest_source_freshness; pub mod ingest_statistics; pub(crate) mod prune_statistics; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index feb5fa1814..c4eee24d5b 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -8,7 +8,7 @@ use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, - SOURCE_FRESHNESS_TABLE_NAME, + RESOURCE_COSTS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, }; use crate::store::errors::ControlError; @@ -286,6 +286,38 @@ pub struct WarehouseObservationRecord { pub observed_at: DateTime, } +/// One estimated query cost observation of a run, resource and warehouse; display only. +/// A refresh appends a later attempt, and readers take the newest attempt per key. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WarehouseResourceCostRecord { + pub state_database: Option, + pub state_schema: String, + pub observation_id: String, + pub run_id: String, + pub orchestration_run_id: Option, + pub attempt: i64, + pub resource_type: String, + pub resource_name: String, + pub warehouse_name: String, + pub warehouse_size: String, + pub warehouse_type: Option, + pub cluster_number: Option, + pub query_count: i64, + pub attributed_seconds: String, + pub bytes_scanned: i64, + pub estimated_credits: String, + pub estimated_usd: Option, + pub status: String, + pub statements_expected: i64, + pub statements_observed: i64, + pub limitations: String, + pub target_name: Option, + pub run_started_at: DateTime, + pub run_completed_at: DateTime, + pub observed_at: DateTime, +} + /// Catalog observations require full scans; only immutable history supports a watermark. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] @@ -301,13 +333,14 @@ pub enum WarehouseHistoryKind { NodeResults, Consumption, Observations, + ResourceCosts, Catalog, Statistics, } impl WarehouseHistoryKind { /// Catalog and statistics stay last: a failed statistics read rewinds one kind, to catalog. - pub const ALL: [Self; 13] = [ + pub const ALL: [Self; 14] = [ Self::Fingerprints, Self::SourceFreshness, Self::AuditResults, @@ -319,6 +352,7 @@ impl WarehouseHistoryKind { Self::NodeResults, Self::Consumption, Self::Observations, + Self::ResourceCosts, Self::Catalog, Self::Statistics, ]; @@ -381,6 +415,11 @@ impl WarehouseHistoryKind { "sqb_warehouse_observations", Some(OBSERVATIONS_TABLE_NAME), ), + Self::ResourceCosts => ( + "resource_costs", + "sqb_warehouse_resource_costs", + Some(RESOURCE_COSTS_TABLE_NAME), + ), Self::Catalog => ("catalog", "sqb_warehouse_catalog", None), Self::Statistics => ("statistics", "sqb_warehouse_statistics", None), } @@ -603,6 +642,7 @@ pub enum WarehouseHistoryRead { NodeResults(NodeResultsRead), Consumption(ConsumptionRead), Observations(StateEventsRead), + ResourceCosts(StateEventsRead), Catalog(CatalogRead), Statistics(StatisticsRead), } @@ -621,6 +661,7 @@ pub enum WarehouseHistoryRecord { NodeResult(Box), Consumption(Box), Observation(Box), + ResourceCost(Box), Catalog(WarehouseCatalogRecord), Statistics(WarehouseStatisticsRecord), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index c21b4bf278..9c9b744be3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -76,6 +76,10 @@ pub(crate) fn finalize_build_cost<'py>( "target_database", native_field(&project, "effective_target_database")?, ), + ( + "target_schema", + native_field(&project, "effective_target_schema")?, + ), ("run_id", native_field(&project, "run_id")?), ("build_status", text(py, cost.status)), ("started_at", cost.started_at), @@ -90,6 +94,12 @@ pub(crate) fn finalize_build_cost<'py>( ("output_stream", context.stream.clone()), ("use_color", boolean(py, context.use_color)), ]; + if let Some(worker) = &context.worker { + fields.push(( + "orchestration_run_id", + text(py, &worker.run_id().to_string()), + )); + } fields.extend(cost.overrides); let finalization: Bound<'py, PyAny> = new_record_with(py, RecordId::BuildCostFinalization, &fields)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 6f4297132c..0ab9b57a92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -30,6 +30,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { freshness::registration::register(module)?; crate::bindings::_helpers::native_host::consumption_state::register(module)?; crate::bindings::_helpers::native_host::observations_state::register(module)?; + crate::bindings::_helpers::native_host::resource_costs_state::register(module)?; crate::bindings::_helpers::native_host::audit_results_state::register(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 45463f934b..974ddb7c6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -59,6 +59,11 @@ impl WorkerBuild { self.execution.claim.request.command } + /// The server run this build executes. + pub(crate) fn run_id(&self) -> uuid::Uuid { + self.execution.claim.request.run_id + } + /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. fn checks_only(&self) -> bool { matches!(self.command(), RunCommand::Test | RunCommand::Audit) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index d41ffba179..51a4c04aff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -14,6 +14,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::cost_report::attribution::report_cost_status; +use crate::bindings::_helpers::cost_report::publication::publish_resource_costs; use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; use crate::bindings::_helpers::cost_telemetry::values::cost_aware; @@ -198,6 +199,38 @@ fn persist<'py>( ) } +/// Record the run's cost in the warehouse state schema; any failure is logged and warned. +fn publish<'py>( + finalization: &Bound<'py, PyAny>, + record: &Bound<'py, PyAny>, + (config, rate_source): (&Bound<'py, PyAny>, &str), +) -> PyResult<()> { + let py: Python<'py> = finalization.py(); + let rate_source: Bound<'py, PyAny> = PyString::new(py, rate_source).into_any(); + let published: PyResult<()> = cost_run(record).and_then(|run| { + publish_resource_costs( + finalization, + &run, + (&native_field(config, "usd_per_credit")?, &rate_source), + ) + }); + let Err(error) = published else { + return Ok(()); + }; + log_failure( + py, + "automatic run-cost warehouse publication failed", + &error, + )?; + warn( + &native_field(finalization, "output_stream")?, + &format!( + "Cost telemetry failed during warehouse publication ({}); build result unchanged.", + error_type(py, &error)? + ), + ) +} + /// `_persist_collection_failure`: log, warn and persist a failed-collection record. fn collection_failure<'py>( finalization: &Bound<'py, PyAny>, @@ -231,6 +264,7 @@ fn collection_failure<'py>( summary(py, status, (config, rate_source), message)?, )?; persist(finalization, &record, "failure artifact persistence")?; + publish(finalization, &record, (config, rate_source))?; Ok(record) } @@ -312,6 +346,9 @@ pub(crate) fn finalize_build_cost<'py>( report_cost_status(&native_field(&cost, "status")?)?; let record: Bound<'py, PyAny> = cost_record(finalization, cost)?; persist(finalization, &record, "artifact persistence")?; + if native_field(finalization, "collect")?.is_truthy()? { + publish(finalization, &record, rate)?; + } if !native_field(finalization, "render")?.is_truthy()? { return Ok(record); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index 854d8e9fb1..b7f6602189 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod attribution; pub(crate) mod degradation; pub(crate) mod exception_types; pub(crate) mod functions; +pub(crate) mod publication; pub(crate) mod records; pub(crate) mod registration; pub(crate) mod store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs new file mode 100644 index 0000000000..85b1822d1a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs @@ -0,0 +1,171 @@ +//! Record each collected run's cost in the target's state schema and re-read incomplete runs. +//! +//! This runs only straight after a build's own cost collection, while the warehouse is awake from +//! the build, so it never resumes an idle warehouse. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refresh; +use sqlbuild_core::cost_command::models::CostCoverage; +use sqlbuild_core::cost_report::models::{CostRun, CostSummary}; +use sqlbuild_state::resource_costs::constants::COLLECTION_FAILED_STATUS; +use sqlbuild_state::resource_costs::main::refresh_candidates::refresh_candidates; +use sqlbuild_state::resource_costs::models::RunCostAttempt; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cost_report::records::summary; +use crate::bindings::_helpers::native_host::consumption_state::Location; +use crate::bindings::_helpers::native_host::resource_costs_state::{ + CostRunIdentity, append_resource_costs, latest_run_costs, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The attempt number of a build's own collection; each accepted re-read appends the next. +const FIRST_ATTEMPT: i64 = 1; +/// Collection budget of each re-read of an older run, which a build waits for. +const REFRESH_BUDGET_SECONDS: f64 = 20.0; +/// The adapter method that marks the resource costs table as supported. +const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql"; + +fn coverage(summary: &CostSummary) -> CostCoverage { + CostCoverage { + status: summary.status.clone(), + expected_statement_count: summary.expected_statement_count, + observed_statement_count: summary.observed_statement_count, + query_count: summary.query_count, + limitations: summary.limitations.clone(), + } +} + +/// The target's state location, when the adapter keeps resource costs and the target has a schema. +fn location(finalization: &Bound<'_, PyAny>) -> PyResult> { + let adapter: Bound<'_, PyAny> = native_field(finalization, "adapter")?; + if !adapter.hasattr(CREATE_TABLE_METHOD)? { + return Ok(None); + } + let Some(schema) = native_field(finalization, "target_schema")?.extract::>()? + else { + return Ok(None); + }; + Ok(Some(Location { + database: native_field(finalization, "target_database")?.extract()?, + schema, + })) +} + +/// Re-collect one older run with the project's current rate. +fn recollect<'py>( + finalization: &Bound<'py, PyAny>, + run: &RunCostAttempt, + (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult { + let py: Python<'py> = finalization.py(); + let parse = imported(py, "datetime", "datetime")?.getattr("fromisoformat")?; + let ledger: Bound<'py, PyAny> = native_field(finalization, "project_dir")? + .call_method1("__truediv__", ("target",))? + .call_method1("__truediv__", ("executions",))? + .call_method1("__truediv__", (run.run_id.as_str(),))? + .call_method1("__truediv__", ("statements.jsonl",))?; + let collected: Bound<'py, PyAny> = call_with( + &native_field(&native_field(finalization, "adapter")?, "collect_run_cost")?, + &[ + ( + "connection_config", + native_field(finalization, "connection_config")?, + ), + ( + "target_database", + native_field(finalization, "target_database")?, + ), + ("run_id", PyString::new(py, &run.run_id).into_any()), + ("started_at", parse.call1((run.run_started_at.as_str(),))?), + ( + "completed_at", + parse.call1((run.run_completed_at.as_str(),))?, + ), + ("statement_ledger_path", ledger), + ("usd_per_credit", usd_per_credit.clone()), + ("rate_source", rate_source.clone()), + ( + "budget_seconds", + REFRESH_BUDGET_SECONDS.into_pyobject(py)?.into_any(), + ), + ], + )?; + summary(&collected) +} + +/// Append the run's collection, then the accepted re-reads of up to a few older incomplete runs. +pub(crate) fn publish_resource_costs( + finalization: &Bound<'_, PyAny>, + run: &CostRun, + rate: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let Some(location) = location(finalization)? else { + return Ok(()); + }; + let py: Python<'_> = finalization.py(); + let adapter: Bound<'_, PyAny> = native_field(finalization, "adapter")?; + let orchestration_run_id: Option = + native_field(finalization, "orchestration_run_id")?.extract()?; + let connection: Bound<'_, PyAny> = adapter.call_method1( + "connect", + (native_field(finalization, "connection_config")?,), + )?; + let published: PyResult<()> = (|| { + append_resource_costs( + &adapter, + &connection, + &location, + ( + &CostRunIdentity { + run_id: &run.run_id, + orchestration_run_id: orchestration_run_id.as_deref(), + target_name: run.target_name.as_deref(), + started_at: &run.started_at, + completed_at: &run.completed_at, + }, + FIRST_ATTEMPT, + ), + &run.cost, + )?; + let now: i64 = imported(py, "time", "time")?.call0()?.extract::()? as i64; + let latest: Vec = latest_run_costs(&adapter, &connection, &location)? + .into_iter() + .filter(|latest| latest.run_id != run.run_id) + .collect(); + for older in refresh_candidates(&latest, now) { + let Ok(refreshed) = recollect(finalization, &older, rate) else { + continue; + }; + let refreshed_coverage: CostCoverage = coverage(&refreshed); + let unchanged: bool = refreshed_coverage == older.coverage + && older.coverage.status != COLLECTION_FAILED_STATUS; + if unchanged || accepts_cost_refresh(&older.coverage, &refreshed_coverage) != Some(true) + { + continue; + } + append_resource_costs( + &adapter, + &connection, + &location, + ( + &CostRunIdentity { + run_id: &older.run_id, + orchestration_run_id: older.orchestration_run_id.as_deref(), + target_name: older.target_name.as_deref(), + started_at: &older.run_started_at, + completed_at: &older.run_completed_at, + }, + older.attempt + 1, + ), + &refreshed, + )?; + } + Ok(()) + })(); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + published?; + closed.map(|_| ()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs index 89f5cb4230..790b011885 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs @@ -62,7 +62,7 @@ fn resource(value: &Bound<'_, PyAny>) -> PyResult { }) } -fn summary(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn summary(value: &Bound<'_, PyAny>) -> PyResult { let mut resources: Vec = Vec::new(); for item in native_field(value, "resources")?.try_iter()? { resources.push(resource(&item?)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 36c6485896..3651363baa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -27,7 +27,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, CONSUMPTION_TABLE_NAME, OBSERVATIONS_TABLE_NAME, - SOURCE_FRESHNESS_TABLE_NAME, + RESOURCE_COSTS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, }; /// The adapter and its open connection. @@ -500,6 +500,7 @@ pub(crate) fn direct_state_prune_candidates<'py>( locations.push((database.clone(), schema.clone(), CONSUMPTION_TABLE_NAME)); locations.push((database.clone(), schema.clone(), OBSERVATIONS_TABLE_NAME)); locations.push((database.clone(), schema.clone(), AUDIT_RESULTS_TABLE_NAME)); + locations.push((database.clone(), schema.clone(), RESOURCE_COSTS_TABLE_NAME)); } let lookup: HashSet = relation_lookup(warehouse, &locations)?; let mut candidates: Vec> = Vec::new(); @@ -525,8 +526,14 @@ pub(crate) fn direct_state_prune_candidates<'py>( AUDIT_RESULTS_TABLE_NAME, "render_prune_audit_results_history_sql", ), + ( + RESOURCE_COSTS_TABLE_NAME, + "render_prune_resource_costs_history_sql", + ), ] { - if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) { + if !lookup_exists(&lookup, (database.as_deref(), schema, table_name)) + || !warehouse.adapter.hasattr(render)? + { continue; } let prune_sql: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 1794e8b508..5a6e327e6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod orchestration_environment; pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; pub(crate) mod registration; +pub(crate) mod resource_costs_state; pub(crate) mod results; pub(crate) mod sqlite_connection; pub(crate) mod state_column_evolution; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs new file mode 100644 index 0000000000..344b5d9f67 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs @@ -0,0 +1,304 @@ +//! DDL, appends, latest reads and pruning of the display-only resource costs table. + +use chrono::DateTime; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::RESOURCE_COSTS_TABLE_NAME; +use sqlbuild_core::cost_command::models::CostCoverage; +use sqlbuild_core::cost_report::models::CostSummary; +use sqlbuild_state::consumption::constants::INTEGER_STATE_VALUE_TYPE; +use sqlbuild_state::resource_costs::constants::{ + NULLABLE_RESOURCE_COST_COLUMNS, RESOURCE_COST_COLUMN_TYPES, RUN_RESOURCE_TYPE, +}; +use sqlbuild_state::resource_costs::main::latest_run_costs_sql::latest_run_costs_sql; +use sqlbuild_state::resource_costs::main::prune_resource_costs_sql::prune_resource_costs_sql; +use sqlbuild_state::resource_costs::main::resource_cost_observation_id::resource_cost_observation_id; +use sqlbuild_state::resource_costs::models::{ResourceCostKey, RunCostAttempt}; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; + +use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::native_host::consumption_state::{ + Location, ROWS_PER_STATEMENT, execute, fixed_state_table_sql, literal, optional_text, + state_table_name, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// Rate source of a price the project configured, the only one shown as USD. +const CONFIGURED_RATE_SOURCE: &str = "configured"; + +/// Create-if-missing DDL for the resource costs table, with adapter-rendered column types. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +fn build_resource_costs_create_table_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + render_framework_type: &Bound<'_, PyAny>, +) -> PyResult { + fixed_state_table_sql( + (database, schema), + (render_qualified_name, render_framework_type), + ( + RESOURCE_COSTS_TABLE_NAME, + &RESOURCE_COST_COLUMN_TYPES, + &NULLABLE_RESOURCE_COST_COLUMNS, + ), + ) +} + +/// SQL deleting observations beyond the newest `retain_versions` per run, resource and warehouse. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] +fn build_resource_costs_prune_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, + retain_versions: i64, +) -> PyResult { + Ok(state_table_name( + render_qualified_name, + (database, schema), + RESOURCE_COSTS_TABLE_NAME, + )? + .map(|table| prune_resource_costs_sql(&table, retain_versions)) + .unwrap_or_default()) +} + +/// The run a collection belongs to, as the run row records it. +pub(crate) struct CostRunIdentity<'a> { + pub(crate) run_id: &'a str, + pub(crate) orchestration_run_id: Option<&'a str>, + pub(crate) target_name: Option<&'a str>, + pub(crate) started_at: &'a str, + pub(crate) completed_at: &'a str, +} + +fn integer(py: Python<'_>, value: Option) -> PyResult { + let value: Bound<'_, PyAny> = value.map_or_else( + || py.None().into_bound(py), + |value| PyInt::new(py, value).into_any(), + ); + render_state_sql_literal(py, &value, INTEGER_STATE_VALUE_TYPE, None) +} + +fn table(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { + let keywords = location.keywords(adapter.py()); + state_table_name( + &native_field(adapter, "render_qualified_name")?, + (&keywords[0].1, &keywords[1].1), + RESOURCE_COSTS_TABLE_NAME, + ) +} + +/// The run row and one row per resource and warehouse of one collection attempt. +fn observation_cells( + py: Python<'_>, + (run, attempt): (&CostRunIdentity<'_>, i64), + summary: &CostSummary, + observed_at: &str, +) -> PyResult>> { + let usd = + |value: &str| (summary.rate_source == CONFIGURED_RATE_SOURCE).then(|| value.to_owned()); + let limitations: String = serde_json::to_string(&summary.limitations) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let run_row = ResourceRow { + resource_type: RUN_RESOURCE_TYPE, + resource_name: run.run_id, + warehouse: ("", "", None, None), + query_count: summary.query_count, + attributed_seconds: &summary.attributed_seconds, + bytes_scanned: summary.bytes_scanned, + estimated_credits: &summary.estimated_compute_credits, + estimated_usd: usd(&summary.estimated_usd), + }; + let mut rows: Vec> = vec![run_row]; + rows.extend(summary.resources.iter().map(|resource| ResourceRow { + resource_type: &resource.resource_type, + resource_name: &resource.resource_name, + warehouse: ( + &resource.warehouse_name, + &resource.warehouse_size, + resource.warehouse_type.as_deref(), + resource.cluster_number, + ), + query_count: resource.query_count, + attributed_seconds: &resource.attributed_seconds, + bytes_scanned: resource.bytes_scanned, + estimated_credits: &resource.estimated_compute_credits, + estimated_usd: usd(&resource.estimated_usd), + })); + rows.iter() + .map(|row| { + let (warehouse_name, warehouse_size, warehouse_type, cluster_number) = row.warehouse; + let observation_id: String = resource_cost_observation_id(&ResourceCostKey { + run_id: run.run_id, + attempt, + resource_type: row.resource_type, + resource_name: row.resource_name, + warehouse_name, + warehouse_size, + warehouse_type, + cluster_number, + }) + .map_err(|error| PyValueError::new_err(format!("{error:?}")))?; + Ok(vec![ + literal(py, Some(&observation_id))?, + literal(py, Some(run.run_id))?, + literal(py, run.orchestration_run_id)?, + integer(py, Some(attempt))?, + literal(py, Some(row.resource_type))?, + literal(py, Some(row.resource_name))?, + literal(py, Some(warehouse_name))?, + literal(py, Some(warehouse_size))?, + literal(py, warehouse_type)?, + integer(py, cluster_number)?, + integer(py, Some(row.query_count))?, + literal(py, Some(row.attributed_seconds))?, + integer(py, Some(row.bytes_scanned))?, + literal(py, Some(row.estimated_credits))?, + literal(py, row.estimated_usd.as_deref())?, + literal(py, Some(&summary.status))?, + integer(py, Some(summary.expected_statement_count))?, + integer(py, Some(summary.observed_statement_count))?, + literal(py, Some(&limitations))?, + literal(py, run.target_name)?, + literal(py, Some(run.started_at))?, + literal(py, Some(run.completed_at))?, + observed_at.to_owned(), + ]) + }) + .collect() +} + +struct ResourceRow<'a> { + resource_type: &'a str, + resource_name: &'a str, + /// Name, size, type and cluster number. + warehouse: (&'a str, &'a str, Option<&'a str>, Option), + query_count: i64, + attributed_seconds: &'a str, + bytes_scanned: i64, + estimated_credits: &'a str, + estimated_usd: Option, +} + +/// Create the schema and table in `location` if missing, then append one collection attempt. +pub(crate) fn append_resource_costs( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, + (run, attempt): (&CostRunIdentity<'_>, i64), + summary: &CostSummary, +) -> PyResult<()> { + let py: Python<'_> = adapter.py(); + let Some(table) = table(adapter, location)? else { + return Ok(()); + }; + let keywords = location.keywords(py); + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(adapter, "render_create_resource_costs_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + let observed_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; + let columns: Vec<&str> = RESOURCE_COST_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + let cells: Vec> = observation_cells(py, (run, attempt), summary, &observed_at)?; + for chunk in cells.chunks(ROWS_PER_STATEMENT) { + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, chunk), + )?; + } + Ok(()) +} + +fn cell_integer(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return value.extract(); + } + optional_text(value)? + .unwrap_or_default() + .trim() + .parse() + .map_err(|_| PyValueError::new_err("resource cost integer cell is not an integer")) +} + +fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { + let cell = |index: usize| row.get_item(index); + let text = |index: usize| -> PyResult { + Ok(optional_text(&cell(index)?)?.unwrap_or_default()) + }; + let limitations: Vec = serde_json::from_str(&text(7)?).unwrap_or_default(); + let run_started_at: String = text(9)?; + let started_epoch: i64 = DateTime::parse_from_rfc3339(run_started_at.trim()) + .map_err(|error| PyValueError::new_err(error.to_string()))? + .timestamp(); + Ok(RunCostAttempt { + run_id: text(0)?, + orchestration_run_id: optional_text(&cell(1)?)?, + attempt: cell_integer(&cell(2)?)?, + coverage: CostCoverage { + status: text(3)?, + expected_statement_count: cell_integer(&cell(4)?)?, + observed_statement_count: cell_integer(&cell(5)?)?, + query_count: cell_integer(&cell(6)?)?, + limitations, + }, + target_name: optional_text(&cell(8)?)?, + run_started_at, + run_completed_at: text(10)?, + started_epoch, + }) +} + +/// The newest run row of each refreshable run in `location`; none when the table is missing. +/// A row that does not parse is skipped, so one bad row never blocks the others. +pub(crate) fn latest_run_costs( + adapter: &Bound<'_, PyAny>, + connection: &Bound<'_, PyAny>, + location: &Location, +) -> PyResult> { + let Some(table) = table(adapter, location)? else { + return Ok(Vec::new()); + }; + let py: Python<'_> = adapter.py(); + let mut keywords = location.keywords(py); + keywords.push(("connection", connection.clone())); + keywords.push(( + "name", + PyString::new(py, RESOURCE_COSTS_TABLE_NAME).into_any(), + )); + let exists: bool = call_with(&native_field(adapter, "get_columns")?, &keywords)?.is_truthy()?; + if !exists { + return Ok(Vec::new()); + } + let result: Bound<'_, PyAny> = execute(adapter, connection, &latest_run_costs_sql(&table))?; + Ok(call_with(&native_field(&result, "fetchall")?, &[])? + .try_iter()? + .filter_map(|row| row.and_then(|row| run_attempt(&row)).ok()) + .collect()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_resource_costs_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_resource_costs_prune_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 18b2a1ef0b..9f9184979d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -12,7 +12,7 @@ use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, - SOURCE_FRESHNESS_TABLE_NAME, + RESOURCE_COSTS_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, }; use sqlbuild_core::models::SqlValue; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; @@ -28,6 +28,7 @@ use sqlbuild_state::freshness::constants::{ OBSERVED_AT_COLUMN, SOURCE_FRESHNESS_COLUMN_TYPES, SQLITE_ADAPTER, }; use sqlbuild_state::observations::constants::OBSERVATION_COLUMN_TYPES; +use sqlbuild_state::resource_costs::constants::RESOURCE_COST_COLUMN_TYPES; use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; @@ -151,7 +152,8 @@ pub(crate) fn read_history<'py>( | WarehouseHistoryRead::OldNameViews(read) | WarehouseHistoryRead::Microbatches(read) | WarehouseHistoryRead::ColumnMigrations(read) - | WarehouseHistoryRead::Observations(read) => &read.location, + | WarehouseHistoryRead::Observations(read) + | WarehouseHistoryRead::ResourceCosts(read) => &read.location, WarehouseHistoryRead::NodeResults(read) => &read.location, WarehouseHistoryRead::Consumption(read) => &read.location, }; @@ -185,6 +187,7 @@ fn read_page<'py>( let freshness_columns = SOURCE_FRESHNESS_COLUMN_TYPES.map(|(name, _)| name); let consumption_columns = CONSUMPTION_COLUMN_TYPES.map(|(name, _)| name); let observation_columns = OBSERVATION_COLUMN_TYPES.map(|(name, _)| name); + let resource_cost_columns = RESOURCE_COST_COLUMN_TYPES.map(|(name, _)| name); let event_columns: Vec = match request { WarehouseHistoryRead::AuditResults(_) => py .import("sqlbuild.executor.audit_results.constants")? @@ -307,6 +310,15 @@ fn read_page<'py>( observation_columns.as_slice(), OBSERVED_AT_COLUMN, ), + WarehouseHistoryRead::ResourceCosts(read) => ( + &read.location, + read.after.as_ref().map(|cursor| cursor.as_slice()), + read.observed_since, + read.limit, + RESOURCE_COSTS_TABLE_NAME, + resource_cost_columns.as_slice(), + OBSERVED_AT_COLUMN, + ), }; let limit = limit.unwrap_or(MAX_HISTORY_RECORDS); if limit == 0 || limit > MAX_HISTORY_RECORDS { @@ -419,7 +431,9 @@ fn read_page<'py>( "input_version", RECORDED_AT_COLUMN, ]), - WarehouseHistoryRead::Observations(_) => Some(&["observation_id", OBSERVED_AT_COLUMN]), + WarehouseHistoryRead::Observations(_) | WarehouseHistoryRead::ResourceCosts(_) => { + Some(&["observation_id", OBSERVED_AT_COLUMN]) + } _ => None, }; let mut cursor_indexes: Vec = columns @@ -579,6 +593,7 @@ fn stream_page<'py>( WarehouseHistoryRecord::NodeResult(record) => record.ts, WarehouseHistoryRecord::Consumption(record) => record.recorded_at, WarehouseHistoryRecord::Observation(record) => record.observed_at, + WarehouseHistoryRecord::ResourceCost(record) => record.observed_at, }; if stream.since.is_none_or(|since| observed >= since) { records.push(record); @@ -651,7 +666,8 @@ fn unchanged_token(request: &WarehouseHistoryRead) -> Option<&str> { | WarehouseHistoryRead::OldNameViews(read) | WarehouseHistoryRead::Microbatches(read) | WarehouseHistoryRead::ColumnMigrations(read) - | WarehouseHistoryRead::Observations(read) => read.unchanged_token.as_deref(), + | WarehouseHistoryRead::Observations(read) + | WarehouseHistoryRead::ResourceCosts(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::NodeResults(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::Consumption(read) => read.unchanged_token.as_deref(), WarehouseHistoryRead::Catalog(_) | WarehouseHistoryRead::Statistics(_) => None, @@ -718,6 +734,13 @@ fn decode_record( | WarehouseHistoryRead::Consumption(ConsumptionRead { location, .. }) => { return decode_event_record(location, row, (columns, &[timestamp])); } + WarehouseHistoryRead::ResourceCosts(read) => { + return decode_event_record( + &read.location, + row, + (columns, &[timestamp, "run_started_at", "run_completed_at"]), + ); + } WarehouseHistoryRead::JanitorEvents(read) => { return decode_event_record( &read.location, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index c34a7e8402..daeac8db78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -813,6 +813,8 @@ const BUILD_COST_FINALIZATION: RecordSpec = RecordSpec { FieldDefault::Text("No executable warehouse work was planned for this build."), ), FieldSpec::new("had_executable_work", FieldDefault::True), + FieldSpec::new("target_schema", FieldDefault::None), + FieldSpec::new("orchestration_run_id", FieldDefault::None), ], }; const CONTRACT_COMMAND_REQUEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 72ebf3aca6..e637523fb6 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -9,4 +9,5 @@ pub mod execution_history; pub mod freshness; pub mod microbatches; pub mod observations; +pub mod resource_costs; pub mod state_tables; diff --git a/crates/sqlbuild-state/src/resource_costs/constants.rs b/crates/sqlbuild-state/src/resource_costs/constants.rs new file mode 100644 index 0000000000..1ea2d48af1 --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/constants.rs @@ -0,0 +1,70 @@ +//! Columns, statuses and bounds of the append-only resource costs state table. + +use crate::consumption::constants::INTEGER_STATE_VALUE_TYPE; +use crate::freshness::constants::{STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE}; + +/// The column the warehouse clock fills on insert. +pub const OBSERVED_AT_COLUMN: &str = "observed_at"; +/// Columns in declaration order with their state value types; `observed_at` is last. +pub const RESOURCE_COST_COLUMN_TYPES: [(&str, &str); 23] = [ + ("observation_id", STRING_STATE_VALUE_TYPE), + ("run_id", STRING_STATE_VALUE_TYPE), + ("orchestration_run_id", STRING_STATE_VALUE_TYPE), + ("attempt", INTEGER_STATE_VALUE_TYPE), + ("resource_type", STRING_STATE_VALUE_TYPE), + ("resource_name", STRING_STATE_VALUE_TYPE), + ("warehouse_name", STRING_STATE_VALUE_TYPE), + ("warehouse_size", STRING_STATE_VALUE_TYPE), + ("warehouse_type", STRING_STATE_VALUE_TYPE), + ("cluster_number", INTEGER_STATE_VALUE_TYPE), + ("query_count", INTEGER_STATE_VALUE_TYPE), + ("attributed_seconds", STRING_STATE_VALUE_TYPE), + ("bytes_scanned", INTEGER_STATE_VALUE_TYPE), + ("estimated_credits", STRING_STATE_VALUE_TYPE), + ("estimated_usd", STRING_STATE_VALUE_TYPE), + ("status", STRING_STATE_VALUE_TYPE), + ("statements_expected", INTEGER_STATE_VALUE_TYPE), + ("statements_observed", INTEGER_STATE_VALUE_TYPE), + ("limitations", STRING_STATE_VALUE_TYPE), + ("target_name", STRING_STATE_VALUE_TYPE), + ("run_started_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), + ("run_completed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), + (OBSERVED_AT_COLUMN, TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; +/// Columns that may be NULL: the server run of a location build, optional warehouse detail, USD +/// without a configured rate, and the target. +pub const NULLABLE_RESOURCE_COST_COLUMNS: [&str; 5] = [ + "orchestration_run_id", + "warehouse_type", + "cluster_number", + "estimated_usd", + "target_name", +]; +/// Resource type of the row carrying a run's totals and status, with empty warehouse columns. +pub const RUN_RESOURCE_TYPE: &str = "run"; +/// Columns a refresh reads from each run's newest run row, in this order. +pub const LATEST_RUN_COST_COLUMNS: [&str; 11] = [ + "run_id", + "orchestration_run_id", + "attempt", + "status", + "statements_expected", + "statements_observed", + "query_count", + "limitations", + "target_name", + "run_started_at", + "run_completed_at", +]; +/// First element of every observation identity. +pub const RESOURCE_COST_TAG: &str = "resource_cost"; +/// Statuses a later build re-reads while the warehouse is awake. +pub const REFRESHABLE_COST_STATUSES: [&str; 3] = ["pending", "partial", "collection_failed"]; +/// Status whose re-reads are bounded by `MAX_FAILED_COLLECTION_ATTEMPTS`. +pub const COLLECTION_FAILED_STATUS: &str = "collection_failed"; +/// Collections of a run, the first included, after which a failed collection is not retried. +pub const MAX_FAILED_COLLECTION_ATTEMPTS: i64 = 3; +/// Runs older than this are not re-read: Snowflake's query history table function keeps 7 days. +pub const REFRESH_WINDOW_DAYS: i64 = 6; +/// Most runs one build re-reads, newest first. +pub const MAX_REFRESHED_RUNS: usize = 3; diff --git a/crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs b/crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs new file mode 100644 index 0000000000..cbcf63e462 --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs @@ -0,0 +1,20 @@ +//! Read the newest run row of every run whose cost a later build may still re-read. + +use crate::resource_costs::constants::{ + LATEST_RUN_COST_COLUMNS, OBSERVED_AT_COLUMN, REFRESHABLE_COST_STATUSES, RUN_RESOURCE_TYPE, +}; + +/// The columns of `LATEST_RUN_COST_COLUMNS` for each run whose newest run row is refreshable. +pub fn latest_run_costs_sql(qualified_name: &str) -> String { + let statuses: String = REFRESHABLE_COST_STATUSES + .iter() + .map(|status| format!("'{status}'")) + .collect::>() + .join(", "); + let columns: String = LATEST_RUN_COST_COLUMNS.join(", "); + format!( + "SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY run_id ORDER BY attempt DESC, \ +{OBSERVED_AT_COLUMN} DESC, observation_id DESC) AS cost_rank FROM {qualified_name} WHERE \ +resource_type = '{RUN_RESOURCE_TYPE}') AS ranked WHERE cost_rank = 1 AND status IN ({statuses})" + ) +} diff --git a/crates/sqlbuild-state/src/resource_costs/main/mod.rs b/crates/sqlbuild-state/src/resource_costs/main/mod.rs new file mode 100644 index 0000000000..90913a32f4 --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/main/mod.rs @@ -0,0 +1,4 @@ +pub mod latest_run_costs_sql; +pub mod prune_resource_costs_sql; +pub mod refresh_candidates; +pub mod resource_cost_observation_id; diff --git a/crates/sqlbuild-state/src/resource_costs/main/prune_resource_costs_sql.rs b/crates/sqlbuild-state/src/resource_costs/main/prune_resource_costs_sql.rs new file mode 100644 index 0000000000..f02752c0e8 --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/main/prune_resource_costs_sql.rs @@ -0,0 +1,14 @@ +//! Prune superseded observations from one resource costs table. + +use crate::resource_costs::constants::OBSERVED_AT_COLUMN; + +/// Delete every observation ranked beyond the newest `retain` per run, resource and warehouse. +pub fn prune_resource_costs_sql(qualified_name: &str, retain: i64) -> String { + format!( + "DELETE FROM {qualified_name} WHERE observation_id IN (SELECT observation_id FROM \ +(SELECT observation_id, ROW_NUMBER() OVER (PARTITION BY run_id, resource_type, resource_name, \ +warehouse_name, warehouse_size, warehouse_type, cluster_number ORDER BY attempt DESC, \ +{OBSERVED_AT_COLUMN} DESC, observation_id DESC) AS cost_rank FROM {qualified_name}) AS ranked \ +WHERE cost_rank > {retain})" + ) +} diff --git a/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs new file mode 100644 index 0000000000..49e7d3cfb2 --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs @@ -0,0 +1,34 @@ +//! Which runs a build re-reads while the warehouse is awake. + +use crate::resource_costs::constants::{ + COLLECTION_FAILED_STATUS, MAX_FAILED_COLLECTION_ATTEMPTS, MAX_REFRESHED_RUNS, + REFRESH_WINDOW_DAYS, REFRESHABLE_COST_STATUSES, +}; +use crate::resource_costs::models::RunCostAttempt; + +const SECONDS_PER_DAY: i64 = 86_400; + +/// Newest first, at most `MAX_REFRESHED_RUNS`: refreshable runs inside the query history window, +/// except failed collections that already used every attempt. `now` is seconds since the epoch. +#[must_use] +pub fn refresh_candidates(latest: &[RunCostAttempt], now: i64) -> Vec { + let oldest: i64 = now - REFRESH_WINDOW_DAYS * SECONDS_PER_DAY; + let mut candidates: Vec = latest + .iter() + .filter(|run| REFRESHABLE_COST_STATUSES.contains(&run.coverage.status.as_str())) + .filter(|run| run.started_epoch >= oldest) + .filter(|run| { + run.coverage.status != COLLECTION_FAILED_STATUS + || run.attempt < MAX_FAILED_COLLECTION_ATTEMPTS + }) + .cloned() + .collect(); + candidates.sort_by(|left, right| { + right + .started_epoch + .cmp(&left.started_epoch) + .then_with(|| left.run_id.cmp(&right.run_id)) + }); + candidates.truncate(MAX_REFRESHED_RUNS); + candidates +} diff --git a/crates/sqlbuild-state/src/resource_costs/main/resource_cost_observation_id.rs b/crates/sqlbuild-state/src/resource_costs/main/resource_cost_observation_id.rs new file mode 100644 index 0000000000..bf111ce406 --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/main/resource_cost_observation_id.rs @@ -0,0 +1,28 @@ +//! `resource_cost_observation_id`: the content-addressed ID of one cost observation. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, optional_text, text}; +use crate::resource_costs::constants::RESOURCE_COST_TAG; +use crate::resource_costs::models::ResourceCostKey; + +/// SHA-256 of the compact JSON array of the key, so a retried append writes the same row. +pub fn resource_cost_observation_id(key: &ResourceCostKey<'_>) -> Result { + let value = JsonValue::Array(vec![ + text(RESOURCE_COST_TAG), + text(key.run_id), + text(&key.attempt.to_string()), + text(key.resource_type), + text(key.resource_name), + text(key.warehouse_name), + text(key.warehouse_size), + optional_text(key.warehouse_type), + optional_text( + key.cluster_number + .map(|number| number.to_string()) + .as_deref(), + ), + ]); + json_digest(&value, false) +} diff --git a/crates/sqlbuild-state/src/resource_costs/mod.rs b/crates/sqlbuild-state/src/resource_costs/mod.rs new file mode 100644 index 0000000000..042268f15e --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/mod.rs @@ -0,0 +1,6 @@ +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/resource_costs/models.rs b/crates/sqlbuild-state/src/resource_costs/models.rs new file mode 100644 index 0000000000..f1d400938d --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/models.rs @@ -0,0 +1,32 @@ +//! The latest collection of a run, as its run row records it. + +use sqlbuild_core::cost_command::models::CostCoverage; + +/// A run row's identity and coverage: the newest attempt per run decides what a refresh does. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunCostAttempt { + pub run_id: String, + /// The server run a location build ran for; None for a build run from the command line. + pub orchestration_run_id: Option, + pub attempt: i64, + pub coverage: CostCoverage, + pub target_name: Option, + /// ISO 8601 texts of the run's window, as recorded. + pub run_started_at: String, + pub run_completed_at: String, + /// Seconds since the epoch of the run's start. + pub started_epoch: i64, +} + +/// What identifies one observation: the run, the collection attempt, the resource and its warehouse. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceCostKey<'a> { + pub run_id: &'a str, + pub attempt: i64, + pub resource_type: &'a str, + pub resource_name: &'a str, + pub warehouse_name: &'a str, + pub warehouse_size: &'a str, + pub warehouse_type: Option<&'a str>, + pub cluster_number: Option, +} diff --git a/crates/sqlbuild-state/src/resource_costs/tests.rs b/crates/sqlbuild-state/src/resource_costs/tests.rs new file mode 100644 index 0000000000..636ab967ae --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_refresh.rs"] +mod refresh; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs new file mode 100644 index 0000000000..36fd823b43 --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs @@ -0,0 +1,122 @@ +use sqlbuild_core::cost_command::models::CostCoverage; + +use crate::resource_costs::main::refresh_candidates::refresh_candidates; +use crate::resource_costs::main::resource_cost_observation_id::resource_cost_observation_id; +use crate::resource_costs::models::{ResourceCostKey, RunCostAttempt}; +use crate::resource_costs::tests::test_types::{ObservationIdTestCase, RefreshTestCase}; + +const NOW: i64 = 1_800_000_000; +const HOUR: i64 = 3_600; +const DAY: i64 = 86_400; + +fn run(run_id: &str, attempt: i64, status: &str, age: i64) -> RunCostAttempt { + RunCostAttempt { + run_id: run_id.to_owned(), + orchestration_run_id: None, + attempt, + coverage: CostCoverage { + status: status.to_owned(), + expected_statement_count: 2, + observed_statement_count: 1, + query_count: 1, + limitations: Vec::new(), + }, + target_name: None, + run_started_at: String::new(), + run_completed_at: String::new(), + started_epoch: NOW - age, + } +} + +#[test] +fn given_latest_run_rows_when_choosing_refreshes_then_only_bounded_recent_incomplete_runs() { + let test_cases = [ + RefreshTestCase { + description: "pending and partial runs are re-read; complete and unavailable ones never", + latest: vec![ + run("pending", 1, "pending", HOUR), + run("partial", 4, "partial", 2 * HOUR), + run("complete", 1, "complete", HOUR), + run("unavailable", 1, "unavailable", HOUR), + ], + expected_runs: vec!["pending", "partial"], + }, + RefreshTestCase { + description: "a failed collection is retried until its third attempt, not after", + latest: vec![ + run("second_failure", 2, "collection_failed", HOUR), + run("third_failure", 3, "collection_failed", HOUR), + ], + expected_runs: vec!["second_failure"], + }, + RefreshTestCase { + description: "runs past Snowflake's query history window are left alone", + latest: vec![ + run("inside", 1, "partial", 6 * DAY - HOUR), + run("outside", 1, "partial", 6 * DAY + HOUR), + ], + expected_runs: vec!["inside"], + }, + RefreshTestCase { + description: "the newest three are re-read first", + latest: (1..=5) + .map(|age| run(&format!("run_{age}"), 1, "pending", age * HOUR)) + .collect(), + expected_runs: vec!["run_1", "run_2", "run_3"], + }, + ]; + for test_case in test_cases { + let runs: Vec = refresh_candidates(&test_case.latest, NOW) + .into_iter() + .map(|run| run.run_id) + .collect(); + assert_eq!(runs, test_case.expected_runs, "{}", test_case.description); + } +} + +fn observation_id(attempt: i64, cluster_number: Option) -> String { + resource_cost_observation_id(&ResourceCostKey { + run_id: "run", + attempt, + resource_type: "model", + resource_name: "orders", + warehouse_name: "BUILD_WH", + warehouse_size: "X-Small", + warehouse_type: Some("STANDARD"), + cluster_number, + }) + .expect("identity encodes") +} + +#[test] +fn given_cost_keys_when_identifying_observations_then_each_attempt_and_cluster_is_its_own_row() { + let base: String = observation_id(1, None); + let test_cases = [ + ObservationIdTestCase { + description: "a retried append of the same attempt writes the same row", + attempt: 1, + cluster_number: None, + expected_same_as_base: true, + }, + ObservationIdTestCase { + description: "a refresh is a new observation", + attempt: 2, + cluster_number: None, + expected_same_as_base: false, + }, + ObservationIdTestCase { + description: "cluster 0 differs from an unknown cluster", + attempt: 1, + cluster_number: Some(0), + expected_same_as_base: false, + }, + ]; + for test_case in test_cases { + let same: bool = observation_id(test_case.attempt, test_case.cluster_number) == base; + assert_eq!( + same, test_case.expected_same_as_base, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/resource_costs/tests/test_types.rs b/crates/sqlbuild-state/src/resource_costs/tests/test_types.rs new file mode 100644 index 0000000000..dfb024dccd --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::resource_costs::models::RunCostAttempt; + +pub(crate) struct RefreshTestCase { + pub(crate) description: &'static str, + pub(crate) latest: Vec, + pub(crate) expected_runs: Vec<&'static str>, +} + +pub(crate) struct ObservationIdTestCase { + pub(crate) description: &'static str, + pub(crate) attempt: i64, + pub(crate) cluster_number: Option, + pub(crate) expected_same_as_base: bool, +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7fa3ba1a37..780316a040 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5524,6 +5524,20 @@ def build_observations_prune_sql( render_qualified_name: Callable[..., str | None], retain_versions: int, ) -> str: ... +def build_resource_costs_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def build_resource_costs_prune_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + retain_versions: int, +) -> str: ... def build_source_freshness_create_table_sql( *, database: str | None, @@ -7243,6 +7257,8 @@ class BuildCostFinalization(NativeRecord): cost_status: CostStatus cost_message: str had_executable_work: bool | None + target_schema: str | None + orchestration_run_id: str | None def __init__( self, project_dir: Path, @@ -7263,6 +7279,8 @@ class BuildCostFinalization(NativeRecord): cost_status: CostStatus = ..., cost_message: str = ..., had_executable_work: bool | None = True, + target_schema: str | None = None, + orchestration_run_id: str | None = None, ) -> None: ... class LineageCommandRequest(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/resource_costs.py b/src/sqlbuild/adapter/contract/classes/resource_costs.py new file mode 100644 index 0000000000..0eb9449207 --- /dev/null +++ b/src/sqlbuild/adapter/contract/classes/resource_costs.py @@ -0,0 +1,37 @@ +"""Adapter implementations for the display-only resource costs table of cost-aware adapters.""" + +from __future__ import annotations + +from typing import Any, cast + + +class ResourceCostsMixin: + """Provide resource-costs-table DDL and pruning for adapters that collect run cost.""" + + def render_create_resource_costs_table_sql(self, *, database: str | None, schema: str) -> str: + """Render DDL that creates the resource costs table when it is missing.""" + + from sqlbuild._native import build_resource_costs_create_table_sql + + adapter: Any = cast(Any, self) + return build_resource_costs_create_table_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + ) + + def render_prune_resource_costs_history_sql( + self, *, database: str | None, schema: str, retain_versions: int + ) -> str: + """Render SQL deleting cost observations beyond the newest `retain_versions` per key.""" + + from sqlbuild._native import build_resource_costs_prune_sql + + adapter: Any = cast(Any, self) + return build_resource_costs_prune_sql( + database=database, + schema=schema, + render_qualified_name=adapter.render_qualified_name, + retain_versions=retain_versions, + ) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index f5eae198f6..6ba1c3a986 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -42,6 +42,7 @@ ) from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin from sqlbuild.adapter.contract.classes.observations import ObservationsMixin +from sqlbuild.adapter.contract.classes.resource_costs import ResourceCostsMixin from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin @@ -203,6 +204,7 @@ class SnowflakeAdapter( MicrobatchMixin, ConsumptionMixin, ObservationsMixin, + ResourceCostsMixin, AuditResultsMixin, UnkeyedDiffMixin, BaseAdapter, @@ -678,6 +680,15 @@ def render_create_observations_table_sql(self, *, database: str | None, schema: schema=schema, ) + def render_create_resource_costs_table_sql(self, *, database: str | None, schema: str) -> str: + return self._with_state_table_retention( + create_sql=super().render_create_resource_costs_table_sql( + database=database, schema=schema + ), + database=database, + schema=schema, + ) + def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: return self._with_state_table_retention( create_sql=super().render_create_microbatch_state_table_sql( From 60eabaad49311e4d216194608564822e0e9389a9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:17:47 +0000 Subject: [PATCH 2/3] wip: serve and show estimated Snowflake cost per run, node and warehouse --- .../orchestration/resource_cost_history.json | 42 +++ clients/typescript/src/schema.d.ts | 280 +++++++++++++++- .../authentication/password_sessions.rs | 12 + .../api/_helpers/resources/person_reads.rs | 92 +++++- .../sqlbuild-orchestration/src/api/models.rs | 11 + .../src/warehouse_history/_helpers/costs.rs | 166 ++++++++++ .../src/warehouse_history/_helpers/mod.rs | 1 + .../src/warehouse_history/main/mod.rs | 3 + .../main/read_person_cost_summary.rs | 72 ++++ .../main/read_person_node_costs.rs | 43 +++ .../main/read_person_run_cost.rs | 45 +++ .../src/warehouse_history/models.rs | 42 +++ .../main/ingest/ingest_resource_costs.rs | 25 +- .../src/warehouse_metadata/models.rs | 3 +- .../tests/api/test_contracts.rs | 2 + .../tests/api/test_resource_costs.rs | 182 ++++++++++ .../tests/api/warehouse_history/helpers.rs | 43 +++ .../_helpers/cost_report/publication.rs | 171 ++++------ .../native_host/resource_costs_state.rs | 311 ++++++++++++------ crates/sqlbuild-state/src/events/main/mod.rs | 1 + .../main/resource_cost_observation_id.rs | 0 .../src/resource_costs/constants.rs | 9 +- .../main/latest_run_costs_sql.rs | 4 +- .../src/resource_costs/main/mod.rs | 1 - .../resource_costs/main/refresh_candidates.rs | 7 +- .../src/resource_costs/tests.rs | 2 + .../src/resource_costs/tests/helpers.rs | 42 +++ .../src/resource_costs/tests/test_refresh.rs | 39 +-- fensu.toml | 1 + .../cli/commands/_helpers/cost/__init__.py | 0 .../cli/commands/_helpers/cost/_test_types.py | 9 + .../cli/commands/_helpers/cost/helpers.py | 68 ++++ .../cost/test_resource_cost_publication.py | 110 +++++++ ui/src/lib/api/client.ts | 9 + ui/src/lib/api/http-client.ts | 29 +- ui/src/lib/api/http-runs.ts | 54 ++- ui/src/lib/api/mock/mock-client.ts | 18 + ui/src/lib/api/mock/operate-store.ts | 125 +++++++ ui/src/lib/api/types/operate.ts | 72 ++++ ui/src/lib/operate/cost.test.ts | 101 ++++++ ui/src/lib/operate/cost.ts | 95 ++++++ ui/src/routes/+page.svelte | 3 + .../routes/catalog/[name]/cost-chart.svelte | 54 +++ ui/src/routes/catalog/[name]/node-view.svelte | 4 +- ui/src/routes/catalog/[name]/runs-tab.svelte | 24 +- ui/src/routes/overview-cost.svelte | 111 +++++++ ui/src/routes/runs/[id]/cost-tab.svelte | 152 +++++++++ ui/src/routes/runs/[id]/run-header.svelte | 11 + ui/src/routes/runs/[id]/run-tabs.svelte | 18 +- ui/src/routes/runs/[id]/state.svelte.ts | 8 +- ui/src/routes/runs/[id]/types.ts | 2 +- ui/src/routes/runs/[id]/utils/index.ts | 1 + ui/src/routes/runs/[id]/utils/tabs.ts | 6 +- .../docs/docs/concepts/adapters/snowflake.mdx | 15 +- 54 files changed, 2448 insertions(+), 303 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/resource_cost_history.json create mode 100644 crates/sqlbuild-orchestration/src/warehouse_history/_helpers/costs.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_cost_summary.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_costs.rs create mode 100644 crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_cost.rs create mode 100644 crates/sqlbuild-orchestration/tests/api/test_resource_costs.rs rename crates/sqlbuild-state/src/{resource_costs => events}/main/resource_cost_observation_id.rs (100%) create mode 100644 crates/sqlbuild-state/src/resource_costs/tests/helpers.rs create mode 100644 tests/integration/src/sqlbuild/cli/commands/_helpers/cost/__init__.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/_helpers/cost/_test_types.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/_helpers/cost/helpers.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/_helpers/cost/test_resource_cost_publication.py create mode 100644 ui/src/lib/operate/cost.test.ts create mode 100644 ui/src/lib/operate/cost.ts create mode 100644 ui/src/routes/catalog/[name]/cost-chart.svelte create mode 100644 ui/src/routes/overview-cost.svelte create mode 100644 ui/src/routes/runs/[id]/cost-tab.svelte diff --git a/benchmarks/duckshop/edge_cases/orchestration/resource_cost_history.json b/benchmarks/duckshop/edge_cases/orchestration/resource_cost_history.json new file mode 100644 index 0000000000..071ad30df0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/resource_cost_history.json @@ -0,0 +1,42 @@ +{ + "record": { + "state_database": null, + "state_schema": "main", + "observation_id": "cost-000", + "run_id": "build-1", + "orchestration_run_id": null, + "attempt": 1, + "resource_type": "model", + "resource_name": "orders", + "warehouse_name": "TRANSFORM_WH", + "warehouse_size": "Small", + "warehouse_type": "STANDARD", + "cluster_number": 1, + "query_count": 2, + "attributed_seconds": "120.000", + "bytes_scanned": 1048576, + "estimated_credits": "0.0667", + "estimated_usd": "0.2000", + "status": "complete", + "statements_expected": 4, + "statements_observed": 4, + "limitations": "[]", + "target_name": "development", + "run_started_at": "2026-01-01T00:00:00Z", + "run_completed_at": "2026-01-01T00:10:00Z", + "observed_at": "2026-01-01T00:15:00Z" + }, + "observations": [ + {"observation_id": "build-1-run-1", "hours_ago": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_type": "_total", "resource_name": "build-1", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "status": "partial", "estimated_credits": "0.15", "estimated_usd": "0.45", "statements_observed": 2}, + {"observation_id": "build-1-orders-1", "hours_ago": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "status": "partial", "estimated_credits": "0.10", "estimated_usd": "0.30", "statements_observed": 2}, + {"observation_id": "build-1-customers-1", "hours_ago": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_name": "customers", "status": "partial", "estimated_credits": "0.05", "estimated_usd": "0.15", "statements_observed": 2}, + {"observation_id": "build-1-run-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_type": "_total", "resource_name": "build-1", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "estimated_credits": "0.40", "estimated_usd": "1.20"}, + {"observation_id": "build-1-orders-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "estimated_credits": "0.25", "estimated_usd": "0.75"}, + {"observation_id": "build-1-setup-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_type": "run", "resource_name": "development", "estimated_credits": "0.05", "estimated_usd": "0.15"}, + {"observation_id": "build-1-customers-2", "hours_ago": 2, "attempt": 2, "orchestration_run_id": "00000000-0000-0000-0000-000000001302", "resource_name": "customers", "estimated_credits": "0.10", "estimated_usd": "0.30"}, + {"observation_id": "build-2-run-1", "hours_ago": 5, "run_id": "build-2", "resource_type": "_total", "resource_name": "build-2", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "estimated_credits": "0.20", "estimated_usd": null}, + {"observation_id": "build-2-orders-1", "hours_ago": 5, "run_id": "build-2", "warehouse_name": "LOAD_WH", "warehouse_size": "X-Small", "estimated_credits": "0.20", "estimated_usd": null}, + {"observation_id": "build-3-run-1", "hours_ago": 240, "run_id": "build-3", "resource_type": "_total", "resource_name": "build-3", "warehouse_name": "", "warehouse_size": "", "warehouse_type": null, "cluster_number": null, "estimated_credits": "5.00", "estimated_usd": "15.00"}, + {"observation_id": "build-3-orders-1", "hours_ago": 240, "run_id": "build-3", "estimated_credits": "5.00", "estimated_usd": "15.00"} + ] +} diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 09e918d5d4..bd7db06b88 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -418,6 +418,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/cost": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonCostSummary"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/fingerprint": { parameters: { query?: never; @@ -546,6 +562,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/cost": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["listPersonNodeCosts"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/runs": { parameters: { query?: never; @@ -642,6 +674,22 @@ export interface paths { patch?: never; trace?: never; }; + "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/cost": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["readPersonRunCost"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/events": { parameters: { query?: never; @@ -1614,6 +1662,48 @@ export interface components { access: components["schemas"]["PersonAccess"]; login_mode: components["schemas"]["LoginMode"]; }; + /** @description Estimated query cost of one group, such as a job, a warehouse size or a node. */ + CostGroup: { + /** Format: double */ + credits: number; + /** @description The group's name; None for builds no server run started, or a resource without a warehouse. */ + key?: string | null; + /** + * Format: int64 + * @description Builds that contributed, counted once each. + */ + runs: number; + /** Format: double */ + seconds: number; + /** + * Format: double + * @description None when any of the group's observations had no configured credit price. + */ + usd?: number | null; + }; + /** @description A target's estimated query cost over a window, from the latest observation of each run. */ + CostSummaryRecord: { + by_job: components["schemas"]["CostGroup"][]; + /** + * @description Every node that ran, keyed `:` like catalog node keys, so + * callers can group nodes by folder. + */ + by_node: components["schemas"]["CostGroup"][]; + by_warehouse_size: components["schemas"]["CostGroup"][]; + /** Format: double */ + credits: number; + /** + * Format: int64 + * @description Runs whose cost is still pending or partial, or whose collection failed. + */ + incomplete_runs: number; + /** Format: int64 */ + runs: number; + /** Format: date-time */ + since: string; + /** Format: double */ + usd?: number | null; + }; /** @description One stretch of the cursor, as raw bound texts. */ CoverageBatch: { end: string; @@ -2044,6 +2134,23 @@ export interface components { MutationResponse: { applied: boolean; }; + /** @description One build's estimated query cost of one node, across every warehouse it used. */ + NodeCostRecord: { + /** Format: double */ + credits: number; + orchestration_run_id?: string | null; + /** Format: int64 */ + query_count: number; + run_id: string; + /** Format: date-time */ + run_started_at: string; + /** Format: double */ + seconds: number; + /** @description The collection status of the whole build. */ + status: string; + /** Format: double */ + usd?: number | null; + }; /** @enum {string} */ NodeEventKind: "success" | "failed" | "recovered"; /** @enum {string} */ @@ -3281,7 +3388,7 @@ export interface components { * @description Catalog observations require full scans; only immutable history supports a watermark. * @enum {string} */ - WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "observations" | "catalog" | "statistics"; + WarehouseHistoryKind: "fingerprints" | "source_freshness" | "audit_results" | "janitor_events" | "migrations" | "old_name_views" | "microbatches" | "column_migrations" | "node_results" | "consumption" | "observations" | "resource_costs" | "catalog" | "statistics"; /** @description Warehouse cleanup facts do not authorize control-plane deletion or worker recovery. */ WarehouseJanitorRecord: { archive_name: string; @@ -3343,6 +3450,46 @@ export interface components { state_schema: string; value: string; }; + /** + * @description One estimated query cost observation of a run, resource and warehouse; display only. + * A refresh appends a later attempt, and readers take the newest attempt per key. + */ + WarehouseResourceCostRecord: { + /** Format: int64 */ + attempt: number; + attributed_seconds: string; + /** Format: int64 */ + bytes_scanned: number; + /** Format: int64 */ + cluster_number?: number | null; + estimated_credits: string; + estimated_usd?: string | null; + limitations: string; + observation_id: string; + /** Format: date-time */ + observed_at: string; + orchestration_run_id?: string | null; + /** Format: int64 */ + query_count: number; + resource_name: string; + resource_type: string; + /** Format: date-time */ + run_completed_at: string; + run_id: string; + /** Format: date-time */ + run_started_at: string; + state_database?: string | null; + state_schema: string; + /** Format: int64 */ + statements_expected: number; + /** Format: int64 */ + statements_observed: number; + status: string; + target_name?: string | null; + warehouse_name: string; + warehouse_size: string; + warehouse_type?: string | null; + }; /** @description Physical namespace of SQLBuild's source-freshness state table, never a source query. */ WarehouseStateLocation: { database?: string | null; @@ -4448,6 +4595,47 @@ export interface operations { }; }; }; + readPersonCostSummary: { + parameters: { + query?: { + /** @description Runs that started in the last this many days; defaults to 7, capped at 90 */ + days?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Estimated query cost by job, warehouse size and node, most expensive first */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CostSummaryRecord"]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; readPersonFingerprint: { parameters: { query: { @@ -4791,6 +4979,50 @@ export interface operations { }; }; }; + listPersonNodeCosts: { + parameters: { + query?: { + /** @description Builds that started in the last this many days; defaults to 7, capped at 90 */ + days?: number; + /** @description Maximum builds; defaults to 100, capped at 1000 */ + limit?: number; + }; + header?: never; + path: { + location_id: string; + target: string; + node_key: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The node's estimated query cost in each build, newest first */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["NodeCostRecord"][]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + }; + }; listPersonNodeRuns: { parameters: { query?: { @@ -5051,6 +5283,52 @@ export interface operations { }; }; }; + readPersonRunCost: { + parameters: { + query?: never; + header?: never; + path: { + location_id: string; + target: string; + run_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The latest estimated query cost observation of the run and of each resource and warehouse it used; empty when none was synced */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WarehouseResourceCostRecord"][]; + }; + }; + /** @description Session rejected */ + 401: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; + /** @description Run not found for a person who may view the whole target */ + 404: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; listPersonRunEvents: { parameters: { query?: { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs index e457021dee..e949cf63da 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/authentication/password_sessions.rs @@ -211,6 +211,18 @@ pub(crate) fn person_routes() -> Router { "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/events", get(crate::api::_helpers::resources::person_reads::run_events), ) + .route( + "/api/control/locations/{location_id}/targets/{target}/cost", + get(crate::api::_helpers::resources::person_reads::cost_summary), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/cost", + get(crate::api::_helpers::resources::person_reads::node_costs), + ) + .route( + "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/cost", + get(crate::api::_helpers::resources::person_reads::run_cost), + ) .route( "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/statements", get(crate::api::_helpers::resources::person_reads::run_statements), diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index 07b107d06c..fadc2e2a04 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -7,7 +7,7 @@ use uuid::Uuid; use crate::api::errors::ApiError; use crate::api::models::{ - PlatformEventsQuery, ReadQuery, RunEventsQuery, RunNodePath, StatementSqlQuery, + CostQuery, PlatformEventsQuery, ReadQuery, RunEventsQuery, RunNodePath, StatementSqlQuery, }; use crate::credentials::models::{AuditContext, AuthenticatedPerson, DefaultRole}; use crate::declarations::main::list_person_jobs::list_person_jobs; @@ -36,14 +36,21 @@ use crate::sensors::models::PersonSensorState; use crate::snapshots::main::list_person_snapshots::list_person_snapshots; use crate::snapshots::models::PublishedSnapshot; use crate::store::models::ControlStore; +use crate::warehouse_history::main::read_person_cost_summary::read_person_cost_summary; +use crate::warehouse_history::main::read_person_node_costs::read_person_node_costs; use crate::warehouse_history::main::read_person_node_runs::read_person_node_runs; +use crate::warehouse_history::main::read_person_run_cost::read_person_run_cost; use crate::warehouse_history::main::read_person_run_events::read_person_run_events; use crate::warehouse_history::main::read_person_run_node_error::read_person_run_node_error; use crate::warehouse_history::main::read_person_run_output::read_person_run_output; use crate::warehouse_history::main::read_person_run_statement_sql::read_person_run_statement_sql; use crate::warehouse_history::main::read_person_run_statements::read_person_run_statements; +use crate::warehouse_history::models::{CostSummaryRecord, NodeCostRecord}; use crate::warehouse_metadata::main::person_reads::list_person_catalog_nodes::list_person_catalog_nodes; -use crate::warehouse_metadata::models::CatalogNodeRecord; +use crate::warehouse_metadata::models::{CatalogNodeRecord, WarehouseResourceCostRecord}; + +/// Default and largest cost window, in days. +const COST_DAYS: (u32, u32) = (7, 90); #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/nodes", operation_id = "listPersonRunNodes", params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path)), @@ -177,6 +184,87 @@ pub(crate) async fn run_statements( .ok_or_else(|| not_found("run not found")) } +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/cost", operation_id = "readPersonRunCost", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path)), + responses((status = 200, body = Vec, description = "The latest estimated query cost observation of the run and of each resource and warehouse it used; empty when none was synced"), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody), + (status = 404, description = "Run not found for a person who may view the whole target")), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn run_cost( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + Path((location_id, target, run_id)): Path<(Uuid, String, Uuid)>, + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + read_person_run_cost( + &store, + (&context, principal), + (location_id, &target, run_id), + default_role, + ) + .await? + .map(Json) + .ok_or_else(|| not_found("run not found")) +} + +fn cost_days(query: &CostQuery) -> u32 { + query.days.unwrap_or(COST_DAYS.0).clamp(1, COST_DAYS.1) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/cost", operation_id = "readPersonCostSummary", + params(("location_id" = Uuid, Path), ("target" = String, Path), + ("days" = Option, Query, description = "Runs that started in the last this many days; defaults to 7, capped at 90")), + responses((status = 200, body = CostSummaryRecord, description = "Estimated query cost by job, warehouse size and node, most expensive first"), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn cost_summary( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target)), Query(options)): (Path<(Uuid, String)>, Query), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result, ApiError> { + Ok(Json( + read_person_cost_summary( + &store, + (&context, principal), + (location_id, &target), + (cost_days(&options), default_role), + ) + .await?, + )) +} + +#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/cost", operation_id = "listPersonNodeCosts", + params(("location_id" = Uuid, Path), ("target" = String, Path), ("node_key" = String, Path), + ("days" = Option, Query, description = "Builds that started in the last this many days; defaults to 7, capped at 90"), + ("limit" = Option, Query, description = "Maximum builds; defaults to 100, capped at 1000")), + responses((status = 200, body = Vec, description = "The node's estimated query cost in each build, newest first"), + (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), + security(("session" = []), ("cliSession" = [])))] +pub(crate) async fn node_costs( + (State(store), Extension(context)): (State, Extension), + Extension(default_role): Extension, + (Path((location_id, target, node_key)), Query(options)): ( + Path<(Uuid, String, String)>, + Query, + ), + Extension(AuthenticatedPerson(principal, _)): Extension, +) -> Result>, ApiError> { + Ok(Json( + read_person_node_costs( + &store, + (&context, principal), + (location_id, &target, &node_key), + ( + cost_days(&options), + options.limit.unwrap_or(100).min(1000), + default_role, + ), + ) + .await?, + )) +} + #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/statements/sql", operation_id = "readPersonRunStatementSql", params(("location_id" = Uuid, Path), ("target" = String, Path), ("run_id" = Uuid, Path), ("statement_id" = String, Query, description = "One statement of this run")), diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index fe53b255bf..5a9d47c5b4 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -33,6 +33,9 @@ use crate::api::_helpers::resources::{ person_reads::node_runs, person_reads::run_events, person_reads::run_statements, + person_reads::run_cost, + person_reads::cost_summary, + person_reads::node_costs, person_reads::run_statement_sql, person_reads::run_output, person_reads::run_node_error, @@ -129,6 +132,14 @@ pub(crate) struct ReadQuery { pub(crate) limit: Option, } +/// A cost read's window in days (default 7, at most 90) and, for history, its row limit. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct CostQuery { + pub(crate) days: Option, + pub(crate) limit: Option, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RunEventsQuery { diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/costs.rs b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/costs.rs new file mode 100644 index 0000000000..9e75933c40 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/costs.rs @@ -0,0 +1,166 @@ +//! The latest estimated query cost observations of a target, each with the job of its server run. + +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_history::models::{CostGroup, NodeCostRecord}; +use crate::warehouse_metadata::models::WarehouseResourceCostRecord; + +/// Total rows carry a build's totals; every other row is one resource on one warehouse. +pub(crate) const TOTAL_RESOURCE_TYPE: &str = "_total"; +/// A build whose cost is final. +pub(crate) const COMPLETE_STATUS: &str = "complete"; + +/// Newest attempt per run, resource and warehouse; a refresh appends, so older attempts stay. +pub(crate) const RUN_COST_SQL: &str = "SELECT record_json FROM (SELECT record_json, ROW_NUMBER() OVER \ + (PARTITION BY run_id, resource_type, resource_name, record_json ->> 'warehouse_name', \ + record_json ->> 'warehouse_size', record_json ->> 'warehouse_type', \ + record_json ->> 'cluster_number' ORDER BY attempt DESC, observed_at DESC, event_id DESC) AS newest \ + FROM sqb_warehouse_resource_costs WHERE location_id=$1 AND target=$2 \ + AND orchestration_run_id=$3) costs WHERE newest=1 \ + ORDER BY record_json ->> 'resource_type', record_json ->> 'resource_name'"; + +/// Newest attempt per run, resource and warehouse since `$3`, optionally of one node (`$4`, `$5`). +const LATEST_COSTS_SQL: &str = "SELECT c.record_json, r.job_key FROM (SELECT record_json, \ + orchestration_run_id, ROW_NUMBER() OVER (PARTITION BY run_id, resource_type, resource_name, \ + record_json ->> 'warehouse_name', record_json ->> 'warehouse_size', \ + record_json ->> 'warehouse_type', record_json ->> 'cluster_number' \ + ORDER BY attempt DESC, observed_at DESC, event_id DESC) AS newest \ + FROM sqb_warehouse_resource_costs WHERE location_id=$1 AND target=$2 AND run_started_at >= $3 \ + AND (CAST($4 AS TEXT) IS NULL OR resource_type IN ('_total', $4)) \ + AND (CAST($5 AS TEXT) IS NULL OR resource_type='_total' OR resource_name=$5)) c \ + LEFT JOIN sqb_runs r ON CAST(r.run_id AS TEXT)=c.orchestration_run_id WHERE c.newest=1"; + +/// One latest observation and the job of the server run that built it, if any. +pub(crate) struct LatestCost { + pub(crate) record: WarehouseResourceCostRecord, + pub(crate) job_key: Option, +} + +/// Latest observations of runs started since `since`; with `node`, only its rows and total rows. +pub(crate) async fn latest_costs_in( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), + since: DateTime, + node: Option<(&str, &str)>, +) -> Result<(ControlTransaction, Vec), ControlError> { + let (transaction, rows) = query( + transaction, + LATEST_COSTS_SQL, + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Time(since), + Parameter::OptionalText(node.map(|(kind, _)| kind.to_owned())), + Parameter::OptionalText(node.map(|(_, name)| name.to_owned())), + ], + ) + .await?; + let costs = rows + .into_iter() + .map(|row| { + Ok(LatestCost { + record: serde_json::from_value(row["record_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + job_key: row["job_key"].as_str().map(str::to_owned), + }) + }) + .collect::>()?; + Ok((transaction, costs)) +} + +/// A decimal text as a number; an unreadable value counts as zero. +pub(crate) fn amount(text: &str) -> f64 { + text.trim().parse().unwrap_or(0.0) +} + +/// Running totals of one cost group. +#[derive(Default)] +pub(crate) struct Accumulator { + credits: f64, + usd: f64, + /// An observation had no configured credit price, so the dollar total would understate. + unpriced: bool, + seconds: f64, + runs: BTreeSet, +} + +impl Accumulator { + pub(crate) fn add(&mut self, cost: &LatestCost) { + let record = &cost.record; + self.credits += amount(&record.estimated_credits); + match record.estimated_usd.as_deref() { + Some(usd) => self.usd += amount(usd), + None => self.unpriced = true, + } + self.seconds += amount(&record.attributed_seconds); + self.runs.insert(record.run_id.clone()); + } + + pub(crate) fn group(self, key: Option) -> CostGroup { + CostGroup { + key, + credits: self.credits, + usd: (!self.unpriced && !self.runs.is_empty()).then_some(self.usd), + seconds: self.seconds, + runs: i64::try_from(self.runs.len()).unwrap_or(i64::MAX), + } + } +} + +/// Groups as records, most expensive first. +pub(crate) fn ranked(groups: BTreeMap, Accumulator>) -> Vec { + let mut ranked: Vec = groups + .into_iter() + .map(|(key, accumulator)| accumulator.group(key)) + .collect(); + ranked.sort_by(|left, right| right.credits.total_cmp(&left.credits)); + ranked +} + +/// The node's rows folded into one record per build with its status, newest first, at most `limit`. +pub(crate) fn node_builds(costs: &[LatestCost], limit: u32) -> Vec { + let mut builds: BTreeMap = BTreeMap::new(); + let mut statuses: BTreeMap = BTreeMap::new(); + for cost in costs { + let record = &cost.record; + if record.resource_type == TOTAL_RESOURCE_TYPE { + statuses.insert(record.run_id.clone(), record.status.clone()); + continue; + } + let build = builds + .entry(record.run_id.clone()) + .or_insert_with(|| NodeCostRecord { + run_id: record.run_id.clone(), + orchestration_run_id: record.orchestration_run_id.clone(), + run_started_at: record.run_started_at, + status: String::new(), + credits: 0.0, + usd: Some(0.0), + seconds: 0.0, + query_count: 0, + }); + build.credits += amount(&record.estimated_credits); + build.usd = build + .usd + .zip(record.estimated_usd.as_deref()) + .map(|(total, usd)| total + amount(usd)); + build.seconds += amount(&record.attributed_seconds); + build.query_count += record.query_count; + } + let mut records: Vec = builds + .into_values() + .map(|mut build| { + build.status = statuses.remove(&build.run_id).unwrap_or_default(); + build + }) + .collect(); + records.sort_by_key(|record| std::cmp::Reverse(record.run_started_at)); + records.truncate(usize::try_from(limit).unwrap_or(usize::MAX)); + records +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/mod.rs index e3f0ef7cbf..60f68fcc8c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod checks; +pub(crate) mod costs; pub(crate) mod coverage; pub(crate) mod maintenance; pub(crate) mod node_runs; diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs index 324d32e1ea..fc5c8f546a 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/mod.rs @@ -3,9 +3,12 @@ pub mod read_person_check; pub mod read_person_check_evidence; pub mod read_person_check_sql; pub mod read_person_checks; +pub mod read_person_cost_summary; pub mod read_person_maintenance; pub mod read_person_microbatch_coverage; +pub mod read_person_node_costs; pub mod read_person_node_runs; +pub mod read_person_run_cost; pub mod read_person_run_events; pub mod read_person_run_node_error; pub mod read_person_run_output; diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_cost_summary.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_cost_summary.rs new file mode 100644 index 0000000000..a8cb29a805 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_cost_summary.rs @@ -0,0 +1,72 @@ +//! A target's estimated query cost over recent days, by job, warehouse size and node. + +use std::collections::BTreeMap; + +use chrono::{Duration, Utc}; +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_history::_helpers::costs::{ + Accumulator, COMPLETE_STATUS, TOTAL_RESOURCE_TYPE, latest_costs_in, ranked, +}; +use crate::warehouse_history::models::{CostGroup, CostSummaryRecord}; + +/// Cost of the runs that started in the last `days` days; empty groups before any cost synced. +pub async fn read_person_cost_summary( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (location_id, target): (Uuid, &str), + (days, default_role): (u32, DefaultRole), +) -> Result { + let since = Utc::now() - Duration::days(i64::from(days)); + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + (context, credentials), + (location_id, target), + default_role, + ) + .await?; + let (transaction, costs) = + latest_costs_in(transaction, (location_id, target), since, None).await?; + commit(transaction).await?; + let mut total = Accumulator::default(); + let mut incomplete_runs: i64 = 0; + let mut by_job: BTreeMap, Accumulator> = BTreeMap::new(); + let mut by_size: BTreeMap, Accumulator> = BTreeMap::new(); + let mut by_node: BTreeMap, Accumulator> = BTreeMap::new(); + for cost in &costs { + let record = &cost.record; + if record.resource_type == TOTAL_RESOURCE_TYPE { + total.add(cost); + incomplete_runs += i64::from(record.status != COMPLETE_STATUS); + by_job.entry(cost.job_key.clone()).or_default().add(cost); + continue; + } + by_size + .entry(Some(record.warehouse_size.clone()).filter(|size| !size.is_empty())) + .or_default() + .add(cost); + by_node + .entry(Some(format!( + "{}:{}", + record.resource_type, record.resource_name + ))) + .or_default() + .add(cost); + } + let total: CostGroup = total.group(None); + Ok(CostSummaryRecord { + since, + credits: total.credits, + usd: total.usd, + runs: total.runs, + incomplete_runs, + by_job: ranked(by_job), + by_warehouse_size: ranked(by_size), + by_node: ranked(by_node), + }) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_costs.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_costs.rs new file mode 100644 index 0000000000..d442f4b87b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_costs.rs @@ -0,0 +1,43 @@ +//! One node's estimated query cost in each recent build, newest first. + +use chrono::{Duration, Utc}; +use uuid::Uuid; + +use crate::authorization::main::authorize_listing::authorize_listing; +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::errors::ControlError; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; +use crate::warehouse_history::_helpers::costs::{latest_costs_in, node_builds}; +use crate::warehouse_history::models::NodeCostRecord; + +/// The node's cost in each build that started in the last `days` days, at most `limit`. +pub async fn read_person_node_costs( + store: &ControlStore, + (context, credentials): (&AuditContext, Uuid), + (location_id, target, node_key): (Uuid, &str, &str), + (days, limit, default_role): (u32, u32, DefaultRole), +) -> Result, ControlError> { + let Some((kind, name)) = node_key.split_once(':') else { + return Err(ControlError::Invalid( + "node key must be :".into(), + )); + }; + let since = Utc::now() - Duration::days(i64::from(days)); + let (transaction, _) = authorize_listing( + begin(store, TransactionMode::Write).await?, + (context, credentials), + (location_id, target), + default_role, + ) + .await?; + let (transaction, costs) = latest_costs_in( + transaction, + (location_id, target), + since, + Some((kind, name)), + ) + .await?; + commit(transaction).await?; + Ok(node_builds(&costs, limit)) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_cost.rs b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_cost.rs new file mode 100644 index 0000000000..69b7cd8a33 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_cost.rs @@ -0,0 +1,45 @@ +//! A run's estimated query cost: the latest observation of each resource and warehouse it used. + +use uuid::Uuid; + +use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::store::errors::ControlError; +use crate::store::main::commit::commit; +use crate::store::main::query::query; +use crate::store::models::{ControlStore, Parameter}; +use crate::warehouse_history::_helpers::costs::RUN_COST_SQL; +use crate::warehouse_history::_helpers::run_recall::begin_run_recall; +use crate::warehouse_metadata::models::WarehouseResourceCostRecord; + +/// `None` when the run is not in this location and target; empty before any cost was synced. +pub async fn read_person_run_cost( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + run: (Uuid, &str, Uuid), + default_role: DefaultRole, +) -> Result>, ControlError> { + let Some((transaction, _, _)) = + begin_run_recall(store, credentials, run, (default_role, None)).await? + else { + return Ok(None); + }; + let (location_id, target, run_id) = run; + let (transaction, rows) = query( + transaction, + RUN_COST_SQL, + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(run_id.to_string()), + ], + ) + .await?; + commit(transaction).await?; + rows.into_iter() + .map(|row| { + serde_json::from_value(row["record_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string())) + }) + .collect::>() + .map(Some) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_history/models.rs b/crates/sqlbuild-orchestration/src/warehouse_history/models.rs index 70aee07613..ffe0787333 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_history/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_history/models.rs @@ -188,3 +188,45 @@ pub struct MicrobatchCoverage { pub struct CoverageQuery { pub limit: Option, } + +/// Estimated query cost of one group, such as a job, a warehouse size or a node. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CostGroup { + /// The group's name; None for builds no server run started, or a resource without a warehouse. + pub key: Option, + pub credits: f64, + /// None when any of the group's observations had no configured credit price. + pub usd: Option, + pub seconds: f64, + /// Builds that contributed, counted once each. + pub runs: i64, +} + +/// A target's estimated query cost over a window, from the latest observation of each run. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CostSummaryRecord { + pub since: DateTime, + pub credits: f64, + pub usd: Option, + pub runs: i64, + /// Runs whose cost is still pending or partial, or whose collection failed. + pub incomplete_runs: i64, + pub by_job: Vec, + pub by_warehouse_size: Vec, + /// Every node that ran, keyed `:` like catalog node keys. + pub by_node: Vec, +} + +/// One build's estimated query cost of one node, across every warehouse it used. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct NodeCostRecord { + pub run_id: String, + pub orchestration_run_id: Option, + pub run_started_at: DateTime, + /// The collection status of the whole build. + pub status: String, + pub credits: f64, + pub usd: Option, + pub seconds: f64, + pub query_count: i64, +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs index 1871724315..04edcfee20 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/ingest/ingest_resource_costs.rs @@ -18,18 +18,7 @@ pub async fn ingest_resource_costs( batch: &MetadataBatch, ) -> Result { let records: Vec = batch_records(batch)?; - let invalid_identity: bool = records.iter().any(|record| { - record.attempt < 1 - || [ - &record.observation_id, - &record.state_schema, - &record.run_id, - &record.resource_type, - &record.resource_name, - ] - .iter() - .any(|value| value.trim().is_empty()) - }); + let invalid_identity: bool = records.iter().any(invalid_identity); ingest_events( store, scope, @@ -69,3 +58,15 @@ pub async fn ingest_resource_costs( ) .await } + +/// Whether an observation lacks an attempt or any part of its identity. +fn invalid_identity(record: &WarehouseResourceCostRecord) -> bool { + let identity: [&String; 5] = [ + &record.observation_id, + &record.state_schema, + &record.run_id, + &record.resource_type, + &record.resource_name, + ]; + record.attempt < 1 || identity.iter().any(|value| value.trim().is_empty()) +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index c4eee24d5b..293157763c 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -286,8 +286,7 @@ pub struct WarehouseObservationRecord { pub observed_at: DateTime, } -/// One estimated query cost observation of a run, resource and warehouse; display only. -/// A refresh appends a later attempt, and readers take the newest attempt per key. +/// One estimated query cost observation of a run, resource and warehouse; refreshes append. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct WarehouseResourceCostRecord { diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index 41553f29dd..b3297708b9 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -86,6 +86,8 @@ mod test_proxy; mod test_publish_encoding; #[path = "test_reads.rs"] mod test_reads; +#[path = "test_resource_costs.rs"] +mod test_resource_costs; #[path = "test_server_errors.rs"] mod test_server_errors; #[path = "test_statistics_recovery.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_resource_costs.rs b/crates/sqlbuild-orchestration/tests/api/test_resource_costs.rs new file mode 100644 index 0000000000..69a4c8e4a5 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_resource_costs.rs @@ -0,0 +1,182 @@ +//! Estimated query cost through the public API, from the latest observation of each run. + +use reqwest::StatusCode; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::api::tests::helpers::stop_api; +use crate::api::tests::test_types::WarehouseMetadataTestCase; +use crate::api::tests::warehouse_history_helpers::{ + batch, cost_groups, cost_observations, fixture, get, login, seed_target, start_password_api, +}; +use crate::runs::tests::helpers::{request, request_run, rows}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::warehouse_metadata::main::ingest::ingest_resource_costs::ingest_resource_costs; + +#[test] +fn given_refreshed_and_unpriced_costs_when_people_read_cost_then_latest_observations_are_grouped() { + let test_cases = [ + WarehouseMetadataTestCase { + description: "SQLite cost reads", + backend: Backend::Sqlite, + expected_observations: 11, + }, + WarehouseMetadataTestCase { + description: "PostgreSQL cost reads", + backend: Backend::Postgres, + expected_observations: 11, + }, + ]; + for test_case in test_cases { + run(async { + let fixture_store = open_test_store(test_case.backend).await.expect("store"); + let store = &fixture_store.store; + let (location, session_id) = seed_target(store).await; + let snapshot_id: Uuid = serde_json::from_value( + rows(store, "SELECT snapshot_id FROM sqb_snapshots", vec![]).await[0] + ["snapshot_id"] + .clone(), + ) + .expect("snapshot"); + let mut daily = request((location, snapshot_id), 0x1302, 0, &["orders", "customers"]); + daily.job_key = Some("daily".into()); + assert!(request_run(store, &daily).await.expect("daily run")); + + let records = cost_observations(&fixture("resource_cost_history.json")); + assert_eq!(records.len(), test_case.expected_observations); + ingest_resource_costs( + store, + (location, "development"), + &batch(session_id, "resource_costs", records), + ) + .await + .expect("real cost ingestion"); + + let (api, client) = start_password_api(store).await; + let viewer = login(&client, &api.url, "orders-viewer").await; + let path = format!("/api/control/locations/{location}/targets/development"); + + let (status, week) = get(&client, (&api.url, &viewer), &format!("{path}/cost")).await; + assert_eq!(status, StatusCode::OK, "{}: {week}", test_case.description); + assert_eq!( + ( + week["runs"].clone(), + week["incomplete_runs"].clone(), + week["usd"].clone() + ), + (json!(2), json!(0), Value::Null), + "the refresh replaced the partial figures, and an unpriced run drops the dollar total" + ); + assert!( + (week["credits"].as_f64().expect("credits") - 0.6).abs() < 1e-9, + "only each run's latest attempt counts, and the run's own statements count once: {week}" + ); + assert_eq!( + cost_groups(&week, "by_job"), + [ + (json!("daily"), 0.4, json!(1.2), 1), + (Value::Null, 0.2, Value::Null, 1) + ], + "the server run's job, and none for a build from the command line" + ); + assert_eq!( + cost_groups(&week, "by_warehouse_size"), + [ + (json!("Small"), 0.4, json!(1.2), 1), + (json!("X-Small"), 0.2, Value::Null, 1) + ] + ); + assert_eq!( + cost_groups(&week, "by_node"), + [ + (json!("model:orders"), 0.45, Value::Null, 2), + (json!("model:customers"), 0.1, json!(0.3), 1), + (json!("run:development"), 0.05, json!(0.15), 1) + ], + "keyed like catalog node keys" + ); + + let (_, month) = get( + &client, + (&api.url, &viewer), + &format!("{path}/cost?days=30"), + ) + .await; + assert_eq!( + month["runs"], + json!(3), + "a longer window reaches older runs" + ); + + let (_, orders) = get( + &client, + (&api.url, &viewer), + &format!("{path}/nodes/model:orders/cost?days=30"), + ) + .await; + let builds: Vec<(Value, Value, Value)> = orders + .as_array() + .expect("builds") + .iter() + .map(|build| { + ( + build["run_id"].clone(), + build["status"].clone(), + build["orchestration_run_id"].clone(), + ) + }) + .collect(); + assert_eq!( + builds, + [ + ( + json!("build-1"), + json!("complete"), + json!(daily.run_id.to_string()) + ), + (json!("build-2"), json!("complete"), Value::Null), + (json!("build-3"), json!("complete"), Value::Null) + ], + "newest first, each build once with its latest status" + ); + assert!((orders[0]["credits"].as_f64().expect("credits") - 0.25).abs() < 1e-9); + + let (_, run_cost) = get( + &client, + (&api.url, &viewer), + &format!("{path}/runs/{}/cost", daily.run_id), + ) + .await; + let mut ids: Vec<&str> = run_cost + .as_array() + .expect("rows") + .iter() + .map(|row| row["observation_id"].as_str().expect("id")) + .collect(); + ids.sort_unstable(); + assert_eq!( + ids, + [ + "build-1-customers-2", + "build-1-orders-2", + "build-1-run-2", + "build-1-setup-2" + ], + "only the latest attempt of each resource" + ); + assert_eq!( + get( + &client, + (&api.url, &viewer), + &format!("{path}/runs/{}/cost", Uuid::now_v7()) + ) + .await + .0, + StatusCode::NOT_FOUND + ); + stop_api(api).await; + close_test_store(fixture_store).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs b/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs index abe809c055..ed60c071aa 100644 --- a/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/warehouse_history/helpers.rs @@ -1,5 +1,6 @@ //! A seeded target with scoped people, password sessions, fixtures and audit reads for warehouse history API tests. +use chrono::{Duration, Utc}; use reqwest::{Client, StatusCode, header}; use serde_json::{Value, json}; use uuid::Uuid; @@ -217,3 +218,45 @@ pub(crate) fn ids(history: &Value, kind: &str) -> Vec { pub(crate) fn coverage_batch(start: &str, end: &str, state: &str, run_id: Value) -> Value { json!({"start": start, "end": end, "state": state, "run_id": run_id}) } + +/// The fixture's observations, each started `hours_ago` before now so the read windows apply. +pub(crate) fn cost_observations(data: &Value) -> Vec { + let now = Utc::now(); + data["observations"] + .as_array() + .expect("observations") + .iter() + .map(|values| { + let mut record = data["record"].clone(); + for (key, value) in values.as_object().expect("override") { + record[key] = value.clone(); + } + let started = now - Duration::hours(values["hours_ago"].as_i64().expect("hours")); + let attempt = record["attempt"].as_i64().expect("attempt"); + record["run_started_at"] = json!(started); + record["run_completed_at"] = json!(started + Duration::minutes(10)); + record["observed_at"] = json!(started + Duration::minutes(10 + 5 * attempt)); + record.as_object_mut().expect("record").remove("hours_ago"); + record + }) + .collect() +} + +/// `(key, credits, usd, runs)` of each cost group, amounts rounded to hundredths. +pub(crate) fn cost_groups(summary: &Value, field: &str) -> Vec<(Value, f64, Value, i64)> { + summary[field] + .as_array() + .expect("groups") + .iter() + .map(|group| { + ( + group["key"].clone(), + (group["credits"].as_f64().expect("credits") * 100.0).round() / 100.0, + group["usd"] + .as_f64() + .map_or(Value::Null, |usd| json!((usd * 100.0).round() / 100.0)), + group["runs"].as_i64().expect("runs"), + ) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs index 85b1822d1a..ff8fae7801 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs @@ -1,11 +1,9 @@ -//! Record each collected run's cost in the target's state schema and re-read incomplete runs. -//! -//! This runs only straight after a build's own cost collection, while the warehouse is awake from -//! the build, so it never resumes an idle warehouse. +//! Record a build's run cost in the target's state schema, then re-read older incomplete runs. -use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyString; +use std::time::{SystemTime, UNIX_EPOCH}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refresh; use sqlbuild_core::cost_command::models::CostCoverage; use sqlbuild_core::cost_report::models::{CostRun, CostSummary}; @@ -13,20 +11,14 @@ use sqlbuild_state::resource_costs::constants::COLLECTION_FAILED_STATUS; use sqlbuild_state::resource_costs::main::refresh_candidates::refresh_candidates; use sqlbuild_state::resource_costs::models::RunCostAttempt; -use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; use crate::bindings::_helpers::cost_report::records::summary; -use crate::bindings::_helpers::native_host::consumption_state::Location; use crate::bindings::_helpers::native_host::resource_costs_state::{ - CostRunIdentity, append_resource_costs, latest_run_costs, + CostRunIdentity, ResourceCostStore, }; use crate::bindings::_helpers::records::fields::field as native_field; /// The attempt number of a build's own collection; each accepted re-read appends the next. const FIRST_ATTEMPT: i64 = 1; -/// Collection budget of each re-read of an older run, which a build waits for. -const REFRESH_BUDGET_SECONDS: f64 = 20.0; -/// The adapter method that marks the resource costs table as supported. -const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql"; fn coverage(summary: &CostSummary) -> CostCoverage { CostCoverage { @@ -38,62 +30,52 @@ fn coverage(summary: &CostSummary) -> CostCoverage { } } -/// The target's state location, when the adapter keeps resource costs and the target has a schema. -fn location(finalization: &Bound<'_, PyAny>) -> PyResult> { - let adapter: Bound<'_, PyAny> = native_field(finalization, "adapter")?; - if !adapter.hasattr(CREATE_TABLE_METHOD)? { - return Ok(None); - } - let Some(schema) = native_field(finalization, "target_schema")?.extract::>()? - else { - return Ok(None); - }; - Ok(Some(Location { - database: native_field(finalization, "target_database")?.extract()?, - schema, - })) +fn unix_now() -> PyResult { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_secs() as i64) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) } -/// Re-collect one older run with the project's current rate. -fn recollect<'py>( - finalization: &Bound<'py, PyAny>, - run: &RunCostAttempt, - (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult { - let py: Python<'py> = finalization.py(); - let parse = imported(py, "datetime", "datetime")?.getattr("fromisoformat")?; - let ledger: Bound<'py, PyAny> = native_field(finalization, "project_dir")? - .call_method1("__truediv__", ("target",))? - .call_method1("__truediv__", ("executions",))? - .call_method1("__truediv__", (run.run_id.as_str(),))? - .call_method1("__truediv__", ("statements.jsonl",))?; - let collected: Bound<'py, PyAny> = call_with( - &native_field(&native_field(finalization, "adapter")?, "collect_run_cost")?, - &[ - ( - "connection_config", - native_field(finalization, "connection_config")?, - ), - ( - "target_database", - native_field(finalization, "target_database")?, - ), - ("run_id", PyString::new(py, &run.run_id).into_any()), - ("started_at", parse.call1((run.run_started_at.as_str(),))?), - ( - "completed_at", - parse.call1((run.run_completed_at.as_str(),))?, - ), - ("statement_ledger_path", ledger), - ("usd_per_credit", usd_per_credit.clone()), - ("rate_source", rate_source.clone()), +/// Append each accepted, changed re-read of the build's older incomplete runs. +fn refresh_older( + store: &ResourceCostStore<'_>, + run_id: &str, + rate: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let latest: Vec = store + .latest()? + .into_iter() + .filter(|latest| latest.run_id != run_id) + .collect(); + for older in refresh_candidates(&latest, unix_now()?) { + let Ok(refreshed) = store + .recollect(&older, rate) + .and_then(|collected| summary(&collected)) + else { + continue; + }; + let refreshed_coverage: CostCoverage = coverage(&refreshed); + let unchanged: bool = refreshed_coverage == older.coverage + && older.coverage.status != COLLECTION_FAILED_STATUS; + if unchanged || accepts_cost_refresh(&older.coverage, &refreshed_coverage) != Some(true) { + continue; + } + store.append( ( - "budget_seconds", - REFRESH_BUDGET_SECONDS.into_pyobject(py)?.into_any(), + &CostRunIdentity { + run_id: &older.run_id, + orchestration_run_id: older.orchestration_run_id.as_deref(), + target_name: older.target_name.as_deref(), + started_at: &older.run_started_at, + completed_at: &older.run_completed_at, + }, + older.attempt + 1, ), - ], - )?; - summary(&collected) + &refreshed, + )?; + } + Ok(()) } /// Append the run's collection, then the accepted re-reads of up to a few older incomplete runs. @@ -102,22 +84,13 @@ pub(crate) fn publish_resource_costs( run: &CostRun, rate: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), ) -> PyResult<()> { - let Some(location) = location(finalization)? else { + let Some(store) = ResourceCostStore::open(finalization)? else { return Ok(()); }; - let py: Python<'_> = finalization.py(); - let adapter: Bound<'_, PyAny> = native_field(finalization, "adapter")?; let orchestration_run_id: Option = native_field(finalization, "orchestration_run_id")?.extract()?; - let connection: Bound<'_, PyAny> = adapter.call_method1( - "connect", - (native_field(finalization, "connection_config")?,), - )?; - let published: PyResult<()> = (|| { - append_resource_costs( - &adapter, - &connection, - &location, + let published: PyResult<()> = store + .append( ( &CostRunIdentity { run_id: &run.run_id, @@ -129,43 +102,9 @@ pub(crate) fn publish_resource_costs( FIRST_ATTEMPT, ), &run.cost, - )?; - let now: i64 = imported(py, "time", "time")?.call0()?.extract::()? as i64; - let latest: Vec = latest_run_costs(&adapter, &connection, &location)? - .into_iter() - .filter(|latest| latest.run_id != run.run_id) - .collect(); - for older in refresh_candidates(&latest, now) { - let Ok(refreshed) = recollect(finalization, &older, rate) else { - continue; - }; - let refreshed_coverage: CostCoverage = coverage(&refreshed); - let unchanged: bool = refreshed_coverage == older.coverage - && older.coverage.status != COLLECTION_FAILED_STATUS; - if unchanged || accepts_cost_refresh(&older.coverage, &refreshed_coverage) != Some(true) - { - continue; - } - append_resource_costs( - &adapter, - &connection, - &location, - ( - &CostRunIdentity { - run_id: &older.run_id, - orchestration_run_id: older.orchestration_run_id.as_deref(), - target_name: older.target_name.as_deref(), - started_at: &older.run_started_at, - completed_at: &older.run_completed_at, - }, - older.attempt + 1, - ), - &refreshed, - )?; - } - Ok(()) - })(); - let closed: PyResult> = adapter.call_method1("close", (&connection,)); + ) + .and_then(|()| refresh_older(&store, &run.run_id, rate)); + let closed: PyResult<()> = store.close(); published?; - closed.map(|_| ()) + closed } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs index 344b5d9f67..871957beca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs @@ -3,23 +3,24 @@ use chrono::DateTime; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyInt, PyString}; +use pyo3::types::{PyFloat, PyInt, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::RESOURCE_COSTS_TABLE_NAME; use sqlbuild_core::cost_command::models::CostCoverage; use sqlbuild_core::cost_report::models::CostSummary; use sqlbuild_state::consumption::constants::INTEGER_STATE_VALUE_TYPE; +use sqlbuild_state::events::main::resource_cost_observation_id::resource_cost_observation_id; use sqlbuild_state::resource_costs::constants::{ - NULLABLE_RESOURCE_COST_COLUMNS, RESOURCE_COST_COLUMN_TYPES, RUN_RESOURCE_TYPE, + NULLABLE_RESOURCE_COST_COLUMNS, RESOURCE_COST_COLUMN_TYPES, TOTAL_RESOURCE_TYPE, }; use sqlbuild_state::resource_costs::main::latest_run_costs_sql::latest_run_costs_sql; use sqlbuild_state::resource_costs::main::prune_resource_costs_sql::prune_resource_costs_sql; -use sqlbuild_state::resource_costs::main::resource_cost_observation_id::resource_cost_observation_id; use sqlbuild_state::resource_costs::models::{ResourceCostKey, RunCostAttempt}; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::execution_history::conversion::{parsed_iso, utc}; use crate::bindings::_helpers::native_host::consumption_state::{ Location, ROWS_PER_STATEMENT, execute, fixed_state_table_sql, literal, optional_text, state_table_name, @@ -28,6 +29,10 @@ use crate::bindings::_helpers::records::fields::field as native_field; /// Rate source of a price the project configured, the only one shown as USD. const CONFIGURED_RATE_SOURCE: &str = "configured"; +/// The adapter method that marks the resource costs table as supported. +const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql"; +/// Collection budget of each re-read of an older run, which a build waits for. +const REFRESH_BUDGET_SECONDS: f64 = 20.0; /// Create-if-missing DDL for the resource costs table, with adapter-rendered column types. #[pyfunction] @@ -93,6 +98,20 @@ fn table(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult { + resource_type: &'a str, + resource_name: &'a str, + warehouse_name: &'a str, + warehouse_size: &'a str, + warehouse_type: Option<&'a str>, + cluster_number: Option, + query_count: i64, + attributed_seconds: &'a str, + bytes_scanned: i64, + estimated_credits: &'a str, + estimated_usd: Option, +} + /// The run row and one row per resource and warehouse of one collection attempt. fn observation_cells( py: Python<'_>, @@ -105,9 +124,12 @@ fn observation_cells( let limitations: String = serde_json::to_string(&summary.limitations) .map_err(|error| PyValueError::new_err(error.to_string()))?; let run_row = ResourceRow { - resource_type: RUN_RESOURCE_TYPE, + resource_type: TOTAL_RESOURCE_TYPE, resource_name: run.run_id, - warehouse: ("", "", None, None), + warehouse_name: "", + warehouse_size: "", + warehouse_type: None, + cluster_number: None, query_count: summary.query_count, attributed_seconds: &summary.attributed_seconds, bytes_scanned: summary.bytes_scanned, @@ -118,12 +140,10 @@ fn observation_cells( rows.extend(summary.resources.iter().map(|resource| ResourceRow { resource_type: &resource.resource_type, resource_name: &resource.resource_name, - warehouse: ( - &resource.warehouse_name, - &resource.warehouse_size, - resource.warehouse_type.as_deref(), - resource.cluster_number, - ), + warehouse_name: &resource.warehouse_name, + warehouse_size: &resource.warehouse_size, + warehouse_type: resource.warehouse_type.as_deref(), + cluster_number: resource.cluster_number, query_count: resource.query_count, attributed_seconds: &resource.attributed_seconds, bytes_scanned: resource.bytes_scanned, @@ -132,16 +152,15 @@ fn observation_cells( })); rows.iter() .map(|row| { - let (warehouse_name, warehouse_size, warehouse_type, cluster_number) = row.warehouse; let observation_id: String = resource_cost_observation_id(&ResourceCostKey { run_id: run.run_id, attempt, resource_type: row.resource_type, resource_name: row.resource_name, - warehouse_name, - warehouse_size, - warehouse_type, - cluster_number, + warehouse_name: row.warehouse_name, + warehouse_size: row.warehouse_size, + warehouse_type: row.warehouse_type, + cluster_number: row.cluster_number, }) .map_err(|error| PyValueError::new_err(format!("{error:?}")))?; Ok(vec![ @@ -151,10 +170,10 @@ fn observation_cells( integer(py, Some(attempt))?, literal(py, Some(row.resource_type))?, literal(py, Some(row.resource_name))?, - literal(py, Some(warehouse_name))?, - literal(py, Some(warehouse_size))?, - literal(py, warehouse_type)?, - integer(py, cluster_number)?, + literal(py, Some(row.warehouse_name))?, + literal(py, Some(row.warehouse_size))?, + literal(py, row.warehouse_type)?, + integer(py, row.cluster_number)?, integer(py, Some(row.query_count))?, literal(py, Some(row.attributed_seconds))?, integer(py, Some(row.bytes_scanned))?, @@ -173,61 +192,6 @@ fn observation_cells( .collect() } -struct ResourceRow<'a> { - resource_type: &'a str, - resource_name: &'a str, - /// Name, size, type and cluster number. - warehouse: (&'a str, &'a str, Option<&'a str>, Option), - query_count: i64, - attributed_seconds: &'a str, - bytes_scanned: i64, - estimated_credits: &'a str, - estimated_usd: Option, -} - -/// Create the schema and table in `location` if missing, then append one collection attempt. -pub(crate) fn append_resource_costs( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, - (run, attempt): (&CostRunIdentity<'_>, i64), - summary: &CostSummary, -) -> PyResult<()> { - let py: Python<'_> = adapter.py(); - let Some(table) = table(adapter, location)? else { - return Ok(()); - }; - let keywords = location.keywords(py); - for statement in - call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? - { - execute(adapter, connection, &statement?.extract::()?)?; - } - let create: String = call_with( - &native_field(adapter, "render_create_resource_costs_table_sql")?, - &keywords, - )? - .extract()?; - if !create.is_empty() { - execute(adapter, connection, &create)?; - } - let observed_at: String = - call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; - let columns: Vec<&str> = RESOURCE_COST_COLUMN_TYPES - .iter() - .map(|(column, _)| *column) - .collect(); - let cells: Vec> = observation_cells(py, (run, attempt), summary, &observed_at)?; - for chunk in cells.chunks(ROWS_PER_STATEMENT) { - execute( - adapter, - connection, - &values_insert_sql(&table, &columns, chunk), - )?; - } - Ok(()) -} - fn cell_integer(value: &Bound<'_, PyAny>) -> PyResult { if value.is_instance_of::() { return value.extract(); @@ -239,14 +203,31 @@ fn cell_integer(value: &Bound<'_, PyAny>) -> PyResult { .map_err(|_| PyValueError::new_err("resource cost integer cell is not an integer")) } +/// A stored run time as RFC 3339 UTC; the table is written in UTC, so a naive value is UTC. +fn utc_text(cell: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = cell.py(); + let value: Bound<'_, PyAny> = match cell.extract::() { + Ok(text) => parsed_iso(py, text.trim())? + .ok_or_else(|| PyValueError::new_err(format!("unreadable run time {text:?}")))?, + Err(_) => cell.clone(), + }; + let aware: Bound<'_, PyAny> = if native_field(&value, "tzinfo")?.is_none() { + call_with(&native_field(&value, "replace")?, &[("tzinfo", utc(py)?)])? + } else { + value.call_method1("astimezone", (utc(py)?,))? + }; + aware.call_method0("isoformat")?.extract() +} + fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { let cell = |index: usize| row.get_item(index); let text = |index: usize| -> PyResult { Ok(optional_text(&cell(index)?)?.unwrap_or_default()) }; - let limitations: Vec = serde_json::from_str(&text(7)?).unwrap_or_default(); - let run_started_at: String = text(9)?; - let started_epoch: i64 = DateTime::parse_from_rfc3339(run_started_at.trim()) + let limitations: Vec = serde_json::from_str(&text(7)?) + .map_err(|error| PyValueError::new_err(format!("unreadable limitations: {error}")))?; + let run_started_at: String = utc_text(&cell(9)?)?; + let started_epoch: i64 = DateTime::parse_from_rfc3339(&run_started_at) .map_err(|error| PyValueError::new_err(error.to_string()))? .timestamp(); Ok(RunCostAttempt { @@ -262,37 +243,159 @@ fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { }, target_name: optional_text(&cell(8)?)?, run_started_at, - run_completed_at: text(10)?, + run_completed_at: utc_text(&cell(10)?)?, started_epoch, }) } -/// The newest run row of each refreshable run in `location`; none when the table is missing. -/// A row that does not parse is skipped, so one bad row never blocks the others. -pub(crate) fn latest_run_costs( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, -) -> PyResult> { - let Some(table) = table(adapter, location)? else { - return Ok(Vec::new()); - }; - let py: Python<'_> = adapter.py(); - let mut keywords = location.keywords(py); - keywords.push(("connection", connection.clone())); - keywords.push(( - "name", - PyString::new(py, RESOURCE_COSTS_TABLE_NAME).into_any(), - )); - let exists: bool = call_with(&native_field(adapter, "get_columns")?, &keywords)?.is_truthy()?; - if !exists { - return Ok(Vec::new()); +/// A target's resource costs table, open on one connection for one build's publication. +pub(crate) struct ResourceCostStore<'py> { + finalization: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + connection: Bound<'py, PyAny>, + location: Location, + table: String, +} + +impl<'py> ResourceCostStore<'py> { + /// Connect when the adapter keeps resource costs and the target has a schema. + pub(crate) fn open(finalization: &Bound<'py, PyAny>) -> PyResult> { + let adapter: Bound<'py, PyAny> = native_field(finalization, "adapter")?; + if !adapter.hasattr(CREATE_TABLE_METHOD)? { + return Ok(None); + } + let Some(schema) = + native_field(finalization, "target_schema")?.extract::>()? + else { + return Ok(None); + }; + let location = Location { + database: native_field(finalization, "target_database")?.extract()?, + schema, + }; + let Some(table) = table(&adapter, &location)? else { + return Ok(None); + }; + let connection: Bound<'py, PyAny> = native_field(&adapter, "connect")? + .call1((native_field(finalization, "connection_config")?,))?; + Ok(Some(Self { + finalization: finalization.clone(), + adapter, + connection, + location, + table, + })) } - let result: Bound<'_, PyAny> = execute(adapter, connection, &latest_run_costs_sql(&table))?; - Ok(call_with(&native_field(&result, "fetchall")?, &[])? + + /// Create the schema and table if missing, then append one collection attempt. + pub(crate) fn append( + &self, + run: (&CostRunIdentity<'_>, i64), + summary: &CostSummary, + ) -> PyResult<()> { + let py: Python<'py> = self.adapter.py(); + let keywords = self.location.keywords(py); + for statement in call_with( + &native_field(&self.adapter, "render_create_schema")?, + &keywords, + )? .try_iter()? - .filter_map(|row| row.and_then(|row| run_attempt(&row)).ok()) - .collect()) + { + self.execute(&statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(&self.adapter, CREATE_TABLE_METHOD)?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + self.execute(&create)?; + } + let observed_at: String = call_with( + &native_field(&self.adapter, "render_current_utc_timestamp")?, + &[], + )? + .extract()?; + let columns: Vec<&str> = RESOURCE_COST_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + let cells: Vec> = observation_cells(py, run, summary, &observed_at)?; + for chunk in cells.chunks(ROWS_PER_STATEMENT) { + self.execute(&values_insert_sql(&self.table, &columns, chunk))?; + } + Ok(()) + } + + /// The newest run row of each refreshable run; none when the table is missing. + pub(crate) fn latest(&self) -> PyResult> { + let py: Python<'py> = self.adapter.py(); + let mut keywords = self.location.keywords(py); + keywords.push(("connection", self.connection.clone())); + keywords.push(( + "name", + PyString::new(py, RESOURCE_COSTS_TABLE_NAME).into_any(), + )); + if !call_with(&native_field(&self.adapter, "get_columns")?, &keywords)?.is_truthy()? { + return Ok(Vec::new()); + } + let result: Bound<'py, PyAny> = self.execute(&latest_run_costs_sql(&self.table))?; + call_with(&native_field(&result, "fetchall")?, &[])? + .try_iter()? + .map(|row| run_attempt(&row?)) + .collect() + } + + /// Re-collect one older run through the adapter, priced at `rate`. + pub(crate) fn recollect( + &self, + run: &RunCostAttempt, + (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + ) -> PyResult> { + let py: Python<'py> = self.adapter.py(); + let parse = imported(py, "datetime", "datetime")?.getattr("fromisoformat")?; + let ledger: Bound<'py, PyAny> = native_field(&self.finalization, "project_dir")? + .call_method1("__truediv__", ("target",))? + .call_method1("__truediv__", ("executions",))? + .call_method1("__truediv__", (run.run_id.as_str(),))? + .call_method1("__truediv__", ("statements.jsonl",))?; + call_with( + &native_field(&self.adapter, "collect_run_cost")?, + &[ + ( + "connection_config", + native_field(&self.finalization, "connection_config")?, + ), + ( + "target_database", + native_field(&self.finalization, "target_database")?, + ), + ("run_id", PyString::new(py, &run.run_id).into_any()), + ("started_at", parse.call1((run.run_started_at.as_str(),))?), + ( + "completed_at", + parse.call1((run.run_completed_at.as_str(),))?, + ), + ("statement_ledger_path", ledger), + ("usd_per_credit", usd_per_credit.clone()), + ("rate_source", rate_source.clone()), + ( + "budget_seconds", + PyFloat::new(py, REFRESH_BUDGET_SECONDS).into_any(), + ), + ], + ) + } + + pub(crate) fn close(self) -> PyResult<()> { + native_field(&self.adapter, "close")? + .call1((&self.connection,)) + .map(|_| ()) + } + + fn execute(&self, sql: &str) -> PyResult> { + execute(&self.adapter, &self.connection, sql) + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs index 459f576db9..a5b6bfbcf2 100644 --- a/crates/sqlbuild-state/src/events/main/mod.rs +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -16,3 +16,4 @@ pub mod old_name_histories; pub mod old_name_view_event_id; pub mod readable_columns; pub mod recorded_renames; +pub mod resource_cost_observation_id; diff --git a/crates/sqlbuild-state/src/resource_costs/main/resource_cost_observation_id.rs b/crates/sqlbuild-state/src/events/main/resource_cost_observation_id.rs similarity index 100% rename from crates/sqlbuild-state/src/resource_costs/main/resource_cost_observation_id.rs rename to crates/sqlbuild-state/src/events/main/resource_cost_observation_id.rs diff --git a/crates/sqlbuild-state/src/resource_costs/constants.rs b/crates/sqlbuild-state/src/resource_costs/constants.rs index 1ea2d48af1..4ce0fe26be 100644 --- a/crates/sqlbuild-state/src/resource_costs/constants.rs +++ b/crates/sqlbuild-state/src/resource_costs/constants.rs @@ -31,8 +31,7 @@ pub const RESOURCE_COST_COLUMN_TYPES: [(&str, &str); 23] = [ ("run_completed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), (OBSERVED_AT_COLUMN, TEXT_TIMESTAMP_STATE_VALUE_TYPE), ]; -/// Columns that may be NULL: the server run of a location build, optional warehouse detail, USD -/// without a configured rate, and the target. +/// Columns that may be NULL: server run, optional warehouse detail, unpriced USD and target. pub const NULLABLE_RESOURCE_COST_COLUMNS: [&str; 5] = [ "orchestration_run_id", "warehouse_type", @@ -40,8 +39,8 @@ pub const NULLABLE_RESOURCE_COST_COLUMNS: [&str; 5] = [ "estimated_usd", "target_name", ]; -/// Resource type of the row carrying a run's totals and status, with empty warehouse columns. -pub const RUN_RESOURCE_TYPE: &str = "run"; +/// Resource type of a run's totals row; executor resource types never start with an underscore. +pub const TOTAL_RESOURCE_TYPE: &str = "_total"; /// Columns a refresh reads from each run's newest run row, in this order. pub const LATEST_RUN_COST_COLUMNS: [&str; 11] = [ "run_id", @@ -68,3 +67,5 @@ pub const MAX_FAILED_COLLECTION_ATTEMPTS: i64 = 3; pub const REFRESH_WINDOW_DAYS: i64 = 6; /// Most runs one build re-reads, newest first. pub const MAX_REFRESHED_RUNS: usize = 3; +/// Seconds in one day, for the refresh window. +pub const SECONDS_PER_DAY: i64 = 86_400; diff --git a/crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs b/crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs index cbcf63e462..64f4752572 100644 --- a/crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs +++ b/crates/sqlbuild-state/src/resource_costs/main/latest_run_costs_sql.rs @@ -1,7 +1,7 @@ //! Read the newest run row of every run whose cost a later build may still re-read. use crate::resource_costs::constants::{ - LATEST_RUN_COST_COLUMNS, OBSERVED_AT_COLUMN, REFRESHABLE_COST_STATUSES, RUN_RESOURCE_TYPE, + LATEST_RUN_COST_COLUMNS, OBSERVED_AT_COLUMN, REFRESHABLE_COST_STATUSES, TOTAL_RESOURCE_TYPE, }; /// The columns of `LATEST_RUN_COST_COLUMNS` for each run whose newest run row is refreshable. @@ -15,6 +15,6 @@ pub fn latest_run_costs_sql(qualified_name: &str) -> String { format!( "SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY run_id ORDER BY attempt DESC, \ {OBSERVED_AT_COLUMN} DESC, observation_id DESC) AS cost_rank FROM {qualified_name} WHERE \ -resource_type = '{RUN_RESOURCE_TYPE}') AS ranked WHERE cost_rank = 1 AND status IN ({statuses})" +resource_type = '{TOTAL_RESOURCE_TYPE}') AS ranked WHERE cost_rank = 1 AND status IN ({statuses})" ) } diff --git a/crates/sqlbuild-state/src/resource_costs/main/mod.rs b/crates/sqlbuild-state/src/resource_costs/main/mod.rs index 90913a32f4..6280159ae3 100644 --- a/crates/sqlbuild-state/src/resource_costs/main/mod.rs +++ b/crates/sqlbuild-state/src/resource_costs/main/mod.rs @@ -1,4 +1,3 @@ pub mod latest_run_costs_sql; pub mod prune_resource_costs_sql; pub mod refresh_candidates; -pub mod resource_cost_observation_id; diff --git a/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs index 49e7d3cfb2..0396acc699 100644 --- a/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs +++ b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs @@ -2,14 +2,11 @@ use crate::resource_costs::constants::{ COLLECTION_FAILED_STATUS, MAX_FAILED_COLLECTION_ATTEMPTS, MAX_REFRESHED_RUNS, - REFRESH_WINDOW_DAYS, REFRESHABLE_COST_STATUSES, + REFRESH_WINDOW_DAYS, REFRESHABLE_COST_STATUSES, SECONDS_PER_DAY, }; use crate::resource_costs::models::RunCostAttempt; -const SECONDS_PER_DAY: i64 = 86_400; - -/// Newest first, at most `MAX_REFRESHED_RUNS`: refreshable runs inside the query history window, -/// except failed collections that already used every attempt. `now` is seconds since the epoch. +/// Newest refreshable runs in the history window, skipping exhausted failures; `now` is epoch seconds. #[must_use] pub fn refresh_candidates(latest: &[RunCostAttempt], now: i64) -> Vec { let oldest: i64 = now - REFRESH_WINDOW_DAYS * SECONDS_PER_DAY; diff --git a/crates/sqlbuild-state/src/resource_costs/tests.rs b/crates/sqlbuild-state/src/resource_costs/tests.rs index 636ab967ae..c1c35620dd 100644 --- a/crates/sqlbuild-state/src/resource_costs/tests.rs +++ b/crates/sqlbuild-state/src/resource_costs/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_refresh.rs"] mod refresh; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs b/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs new file mode 100644 index 0000000000..49237b71ee --- /dev/null +++ b/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs @@ -0,0 +1,42 @@ +//! Run attempts and observation IDs shared by the resource cost tests. + +use sqlbuild_core::cost_command::models::CostCoverage; + +use crate::events::main::resource_cost_observation_id::resource_cost_observation_id; +use crate::resource_costs::models::{ResourceCostKey, RunCostAttempt}; + +/// The fixed clock every refresh case is judged at, in epoch seconds. +pub(crate) const NOW: i64 = 1_800_000_000; + +pub(crate) fn run(run_id: &str, attempt: i64, status: &str, age: i64) -> RunCostAttempt { + RunCostAttempt { + run_id: run_id.to_owned(), + orchestration_run_id: None, + attempt, + coverage: CostCoverage { + status: status.to_owned(), + expected_statement_count: 2, + observed_statement_count: 1, + query_count: 1, + limitations: Vec::new(), + }, + target_name: None, + run_started_at: String::new(), + run_completed_at: String::new(), + started_epoch: NOW - age, + } +} + +pub(crate) fn observation_id(attempt: i64, cluster_number: Option) -> String { + resource_cost_observation_id(&ResourceCostKey { + run_id: "run", + attempt, + resource_type: "model", + resource_name: "orders", + warehouse_name: "BUILD_WH", + warehouse_size: "X-Small", + warehouse_type: Some("STANDARD"), + cluster_number, + }) + .expect("identity encodes") +} diff --git a/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs index 36fd823b43..fb9ef8b77c 100644 --- a/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs +++ b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs @@ -1,33 +1,10 @@ -use sqlbuild_core::cost_command::models::CostCoverage; - use crate::resource_costs::main::refresh_candidates::refresh_candidates; -use crate::resource_costs::main::resource_cost_observation_id::resource_cost_observation_id; -use crate::resource_costs::models::{ResourceCostKey, RunCostAttempt}; +use crate::resource_costs::tests::helpers::{NOW, observation_id, run}; use crate::resource_costs::tests::test_types::{ObservationIdTestCase, RefreshTestCase}; -const NOW: i64 = 1_800_000_000; const HOUR: i64 = 3_600; const DAY: i64 = 86_400; -fn run(run_id: &str, attempt: i64, status: &str, age: i64) -> RunCostAttempt { - RunCostAttempt { - run_id: run_id.to_owned(), - orchestration_run_id: None, - attempt, - coverage: CostCoverage { - status: status.to_owned(), - expected_statement_count: 2, - observed_statement_count: 1, - query_count: 1, - limitations: Vec::new(), - }, - target_name: None, - run_started_at: String::new(), - run_completed_at: String::new(), - started_epoch: NOW - age, - } -} - #[test] fn given_latest_run_rows_when_choosing_refreshes_then_only_bounded_recent_incomplete_runs() { let test_cases = [ @@ -74,20 +51,6 @@ fn given_latest_run_rows_when_choosing_refreshes_then_only_bounded_recent_incomp } } -fn observation_id(attempt: i64, cluster_number: Option) -> String { - resource_cost_observation_id(&ResourceCostKey { - run_id: "run", - attempt, - resource_type: "model", - resource_name: "orders", - warehouse_name: "BUILD_WH", - warehouse_size: "X-Small", - warehouse_type: Some("STANDARD"), - cluster_number, - }) - .expect("identity encodes") -} - #[test] fn given_cost_keys_when_identifying_observations_then_each_attempt_and_cluster_is_its_own_row() { let base: String = observation_id(1, None); diff --git a/fensu.toml b/fensu.toml index 339939297d..394987ee96 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2885,6 +2885,7 @@ module_layers = [ "native_host/observations_state=front", "native_host/orchestration_sensors=commands", "native_host/orchestration_triggers=project", + "native_host/resource_costs_state=front", "native_host/warehouse_metadata=commands", "native_threads=infra", "observations=plan", diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/__init__.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/_test_types.py new file mode 100644 index 0000000000..b4d609aa39 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/_test_types.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass +from decimal import Decimal + + +@dataclass(frozen=True) +class ResourceCostPublicationTestCase: + description: str + usd_per_credit_is_default: bool + expected_run_usd: Decimal | None diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/helpers.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/helpers.py new file mode 100644 index 0000000000..873feef147 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/helpers.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from collections.abc import Callable +from decimal import Decimal +from typing import Any + +from sqlbuild._native import ResourceCost +from sqlbuild.adapter.contract.classes.resource_costs import ResourceCostsMixin +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.cost.models import RunCostSummary +from sqlbuild.cost.types import CostCapability, CostStatus + + +def scripted_summary( + status: CostStatus, observed: int, credits: str +) -> Callable[[Decimal, str], RunCostSummary]: + """A collection of four statements, `observed` of them seen, all on one model and warehouse.""" + + def collected(usd_per_credit: Decimal, rate_source: str) -> RunCostSummary: + amount: Decimal = Decimal(credits) + return RunCostSummary( + status=status, + usd_per_credit=usd_per_credit, + rate_source=rate_source, + resources=( + ResourceCost( + resource_type="model", + resource_name="orders", + warehouse_name="TRANSFORM_WH", + warehouse_size="Small", + warehouse_type="STANDARD", + cluster_number=1, + query_count=observed, + attributed_seconds=Decimal(observed * 30), + bytes_scanned=observed * 1024, + estimated_compute_credits=amount, + estimated_usd=amount * usd_per_credit, + ), + ), + query_count=observed, + attributed_seconds=Decimal(observed * 30), + bytes_scanned=observed * 1024, + estimated_compute_credits=amount, + estimated_usd=amount * usd_per_credit, + expected_statement_count=4, + observed_statement_count=observed, + missing_statement_count=4 - observed, + ) + + return collected + + +class ScriptedCostAdapter(ResourceCostsMixin, DuckDbAdapter): + """DuckDB keeps the state table; each run's collections are scripted in order.""" + + def __init__( + self, collections: dict[str, list[Callable[[Decimal, str], RunCostSummary]]] + ) -> None: + self.collections = collections + self.collected: list[str] = [] + + def cost_capability(self) -> CostCapability: + return CostCapability.SNOWFLAKE_QUERY_HISTORY + + def collect_run_cost(self, **kwargs: Any) -> RunCostSummary: + run_id: str = kwargs["run_id"] + self.collected.append(run_id) + return self.collections[run_id].pop(0)(kwargs["usd_per_credit"], kwargs["rate_source"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/test_resource_cost_publication.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/test_resource_cost_publication.py new file mode 100644 index 0000000000..db578fa8e7 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/cost/test_resource_cost_publication.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from io import StringIO +from pathlib import Path +from typing import Any + +import duckdb +import pytest + +from sqlbuild._native import BuildCostFinalization, CostConfig, finalize_build_cost +from sqlbuild.cost.types import CostStatus +from tests.integration.src.sqlbuild.cli.commands._helpers.cost._test_types import ( + ResourceCostPublicationTestCase, +) +from tests.integration.src.sqlbuild.cli.commands._helpers.cost.helpers import ( + ScriptedCostAdapter, + scripted_summary, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ResourceCostPublicationTestCase( + description="configured price records dollars", + usd_per_credit_is_default=False, + expected_run_usd=Decimal("0.36"), + ), + ResourceCostPublicationTestCase( + description="default price records credits only", + usd_per_credit_is_default=True, + expected_run_usd=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_later_builds_when_publishing_cost_then_only_improved_refreshes_are_appended( + test_case: ResourceCostPublicationTestCase, + tmp_path: Path, +) -> None: + warehouse: Path = tmp_path / "warehouse.duckdb" + adapter: ScriptedCostAdapter = ScriptedCostAdapter( + { + "run-1": [ + scripted_summary(CostStatus.PARTIAL, 2, "0.06"), + scripted_summary(CostStatus.COMPLETE, 4, "0.12"), + ], + "run-2": [scripted_summary(CostStatus.COMPLETE, 4, "0.10")], + "run-3": [ + scripted_summary(CostStatus.PARTIAL, 3, "0.09"), + scripted_summary(CostStatus.PARTIAL, 2, "0.06"), + ], + "run-4": [scripted_summary(CostStatus.COMPLETE, 4, "0.11")], + } + ) + started: datetime = datetime.now(UTC) - timedelta(hours=1) + for index, run_id in enumerate(("run-1", "run-2", "run-3", "run-4")): + _ = finalize_build_cost( + BuildCostFinalization( + project_dir=tmp_path, + adapter_name="snowflake", + adapter=adapter, + connection_config={"database": str(warehouse)}, + target_name="dev", + target_database="warehouse", + run_id=run_id, + build_status="success", + started_at=started + timedelta(minutes=10 * index), + completed_at=started + timedelta(minutes=10 * index + 5), + config=CostConfig( + usd_per_credit=Decimal("3"), + usd_per_credit_is_default=test_case.usd_per_credit_is_default, + ), + output_stream=StringIO(), + use_color=False, + collect=True, + render=False, + had_executable_work=True, + target_schema="main", + orchestration_run_id=None, + ) + ) + + with duckdb.connect(str(warehouse), read_only=True) as connection: + runs: list[tuple[Any, ...]] = connection.execute( + "SELECT resource_name, attempt, status, statements_observed FROM main._sqlbuild_resource_costs " + "WHERE resource_type = '_total' ORDER BY resource_name, attempt" + ).fetchall() + refreshed: tuple[Any, ...] | None = connection.execute( + "SELECT CAST(estimated_usd AS DECIMAL(18, 2)), CAST(estimated_credits AS DECIMAL(18, 2)) " + "FROM main._sqlbuild_resource_costs " + "WHERE resource_type = '_total' AND resource_name = 'run-1' AND attempt = 2" + ).fetchone() + models: int = connection.execute( + "SELECT COUNT(*) FROM main._sqlbuild_resource_costs WHERE resource_type = 'model'" + ).fetchone()[0] + + assert runs == [ + ("run-1", 1, "partial", 2), + ("run-1", 2, "complete", 4), + ("run-2", 1, "complete", 4), + ("run-3", 1, "partial", 3), + ("run-4", 1, "complete", 4), + ], "the partial run-1 is refreshed once, and run-3's regressed re-read is not appended" + assert adapter.collected == ["run-1", "run-2", "run-1", "run-3", "run-4", "run-3"] + assert models == len(runs), "each attempt also records its model row" + assert refreshed is not None + assert refreshed == (test_case.expected_run_usd, Decimal("0.12")) diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index 4bc5c5caeb..7dbdfec6fd 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -28,6 +28,9 @@ import type { RunRowPage, RunRowQuery, RunStatement, + RunCostRow, + CostSummary, + NodeBuildCost, Snapshot, RunTabCounts, Scope, @@ -95,6 +98,12 @@ export interface ApiClient extends PlatformApi { /** Recall events and scheduler facts in storage order, paged by `after`. */ listRunEvents(scope: Scope, runId: string, query?: { after?: number; limit?: number }): Promise; listRunStatements(scope: Scope, runId: string): Promise; + /** The latest estimated query cost observations of one run; empty until a collection synced. */ + listRunCost(scope: Scope, runId: string): Promise; + /** Estimated query cost of the runs that started in the last `days` days, by job, warehouse size and node. */ + getCostSummary(scope: Scope, days: number): Promise; + /** One node's estimated query cost per build over the last `days` days, newest first. */ + listNodeCosts(scope: Scope, nodeKey: string, days: number): Promise; /** One statement's SQL, fetched when its row is expanded. `view.sql` */ getStatementSql(scope: Scope, runId: string, statementId: string): Promise; listRunOutput(scope: Scope, runId: string): Promise; diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index ef45f56d6e..bbc74eb99a 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -16,12 +16,12 @@ import { toPlatformEventRecords, toPlatformEvents, toPlatformHealth } from './ht import { roleDetailOf, toRoleDefinition } from './http-roles'; import { jobDetailOf, runWindowOf, toJob, toTick, toTrigger, toTriggerStateEvent, triggerDetailOf } from './http-jobs'; import { timelineOf } from './http-timeline'; -import { RUN_WINDOW, groupRunNodes, parseRunCursor, planResultOf, queueEntriesOf, toAdhocCheck, runDetailOf, runRowPageOver, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements } from './http-runs'; +import { RUN_WINDOW, groupRunNodes, parseRunCursor, planResultOf, queueEntriesOf, toAdhocCheck, runDetailOf, runRowPageOver, runsMatching, runTabCountsOf, toRunNode, toRuns, toRunEvent, toRunOutput, toRunStatements, toRunCostRows, toCostSummary, toNodeBuildCosts } from './http-runs'; import type { ChangeSubscription, ChangedChannel, ChangedSignal, ConnectionState } from './signals'; import type { AuthMode, Bootstrap, Capability, Location, NavCounts, PlatformEvent, Scope, Snapshot, WarehouseSync } from './types'; import type { Account, AuditEntry, AuditQuery, ConnectionTest, LocationDetail, ServerSettings, WarehouseConnection, LocationSummary, PlatformEventQuery, PlatformEventRecord, PlatformHealth, MaintenanceHistory, ProviderConnection, RoleAssignment, RoleAssignmentRequest, RoleDefinition, RoleDetail } from './types/platform'; import type { Acknowledgement, CatalogNode, CatalogNodeDetail, Check, CheckDetail, CheckEvidence, CheckSql, Overview, SnapshotChange, ColumnLineage } from './types/build'; -import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, StatementSql } from './types/operate'; +import type { AdhocRunCheck, AdhocRunRequest, BulkRunResult, PlanResult, QueueEntry, Timeline, Job, JobDetail, JobLaunchOptions, RerunMode, SnapshotNode, Run, RunDetail, RunListQuery, RunRowPage, RunRowQuery, RunTabCounts, Trigger, TriggerDetail, TriggerState, RunEventPage, RunNodeError, RunOutputLine, RunStatement, RunCostRow, CostSummary, NodeBuildCost, StatementSql } from './types/operate'; type Wire = components['schemas']; @@ -41,6 +41,9 @@ const SUPPORTED: ReadonlySet = new Set([ 'timeline', 'listRunEvents', 'listRunStatements', + 'listRunCost', + 'getCostSummary', + 'listNodeCosts', 'getStatementSql', 'listRunOutput', 'getRunNodeError', @@ -383,6 +386,28 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, if (!data) throw new ApiError(errorText(error, response), response.status); return toRunStatements(data, runId); }, + async listRunCost(scope: Scope, runId: string): Promise { + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/runs/{run_id}/cost', { params: { path: runPath(scope, runId) } }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toRunCostRows(data); + }, + async getCostSummary(scope: Scope, days: number): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/cost', { params: { path: { location_id: locationId, target }, query: { days } } }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toCostSummary(data); + }, + async listNodeCosts(scope: Scope, nodeKey: string, days: number): Promise { + const { locationId, target } = parseLocationKey(scope.locationId); + const { data, error, response } = await http.GET('/api/control/locations/{location_id}/targets/{target}/nodes/{node_key}/cost', { + params: { path: { location_id: locationId, target, node_key: nodeKey }, query: { days } } + }); + if (response.status === 401) onUnauthorized(); + if (!data) throw new ApiError(errorText(error, response), response.status); + return toNodeBuildCosts(data); + }, async getStatementSql(scope: Scope, runId: string, statementId: string): Promise { const refusal = `${scope.locationId}|${runId}`; if (!sqlLocations.has(scope.locationId) || sqlRefused.has(refusal)) throw new ApiError('You may not view the SQL of this run', 403); diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index d5f6d4a9e8..22eb674b7b 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -1,7 +1,7 @@ import type { components } from '$control/schema.d.ts'; import { isActiveRun } from '$lib/status'; import type { Capability, Principal } from './types'; -import type { AdhocRunCheck, LaunchVerb, NodeCounts, PlanDocument, PlanResult, QueueEntry, QueueWait, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement } from './types/operate'; +import type { AdhocRunCheck, LaunchVerb, NodeCounts, PlanDocument, PlanResult, QueueEntry, QueueWait, Run, RunDetail, RunListQuery, RunNode, RunNodeStatus, RunRow, RunRowPage, RunRowQuery, RunTabCounts, RunEvent, RunEventLevel, RunOutputLine, RunStatement, RunCostRow, CostGroup, CostSummary, NodeBuildCost } from './types/operate'; import { RECALL_EVENT_TYPES, SCHEDULER_FACTS } from './types/operate'; type Wire = components['schemas']; @@ -408,3 +408,55 @@ export function toRunOutput(wire: Wire['RunOutput']): RunOutputLine[] { return [...lines, ...errors].sort((a, b) => a.at.localeCompare(b.at)); } +const COST_STATUSES: RunCostRow['status'][] = ['complete', 'partial', 'pending', 'collection_failed', 'unavailable']; + +const costStatus = (status: string): RunCostRow['status'] => COST_STATUSES.find((s) => s === status) ?? 'unavailable'; + +const toCostGroup = (g: Wire['CostGroup']): CostGroup => ({ key: g.key ?? null, credits: g.credits, usd: g.usd ?? null, seconds: g.seconds, runs: g.runs }); + +export function toCostSummary(wire: Wire['CostSummaryRecord']): CostSummary { + return { + since: wire.since, + credits: wire.credits, + usd: wire.usd ?? null, + runs: wire.runs, + incompleteRuns: wire.incomplete_runs, + byJob: wire.by_job.map(toCostGroup), + byWarehouseSize: wire.by_warehouse_size.map(toCostGroup), + byNode: wire.by_node.map(toCostGroup) + }; +} + +export function toNodeBuildCosts(wire: Wire['NodeCostRecord'][]): NodeBuildCost[] { + return wire.map((c) => ({ + buildRunId: c.run_id, + runId: c.orchestration_run_id ?? null, + startedAt: c.run_started_at, + status: costStatus(c.status), + credits: c.credits, + usd: c.usd ?? null, + seconds: c.seconds, + queryCount: c.query_count + })); +} + +/** Latest cost observations as numbers; a status the UI does not know reads as unavailable. */ +export function toRunCostRows(wire: Wire['WarehouseResourceCostRecord'][]): RunCostRow[] { + return wire.map((r) => ({ + buildRunId: r.run_id, + resourceType: r.resource_type, + resourceName: r.resource_name, + // Catalog node keys are `kind:name`, and a resource's type is its node kind. + nodeKey: r.resource_type === '_total' ? null : `${r.resource_type}:${r.resource_name}`, + warehouseName: r.warehouse_name, + warehouseSize: r.warehouse_size, + queryCount: r.query_count, + seconds: Number(r.attributed_seconds), + bytesScanned: r.bytes_scanned, + credits: Number(r.estimated_credits), + usd: r.estimated_usd == null ? null : Number(r.estimated_usd), + status: costStatus(r.status), + statementsExpected: r.statements_expected, + statementsObserved: r.statements_observed + })); +} diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index 97c330805e..c1231361e4 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -23,6 +23,9 @@ import type { RunNodeError, RunOutputLine, RunStatement, + RunCostRow, + CostSummary, + NodeBuildCost, Scope, Snapshot, SnapshotNode, @@ -212,6 +215,21 @@ export class MockApiClient implements ApiClient { return this.#operate.listRunStatements(scope, runId); } + async listRunCost(scope: Scope, runId: string): Promise { + this.#location(scope); + return this.#operate.listRunCost(scope, runId); + } + + async getCostSummary(scope: Scope, days: number): Promise { + this.#location(scope); + return this.#operate.getCostSummary(scope, days); + } + + async listNodeCosts(scope: Scope, nodeKey: string, days: number): Promise { + this.#location(scope); + return this.#operate.listNodeCosts(scope, nodeKey, days); + } + async listRunOutput(scope: Scope, runId: string): Promise { this.#location(scope); return this.#operate.listRunOutput(scope, runId); diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index ec6a9e1ec3..dda96a7653 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -33,6 +33,9 @@ import type { RunRowPage, RunRowQuery, RunStatement, + RunCostRow, + CostSummary, + NodeBuildCost, RunTabCounts, Scope, SnapshotNode, @@ -314,6 +317,128 @@ export class MockOperate { return this.#ctx.reply(execution(rec, this.#nodesOf(rec, now), now).statements); } + listRunCost(scope: Scope, runId: string): Promise { + return this.#ctx.reply(this.#costRows(scope, this.#require(runId), Date.now())); + } + + /** The latest cost of the builds that started in the window, grouped as the server groups them. */ + getCostSummary(scope: Scope, days: number): Promise { + const now = Date.now(); + const since = now - days * 86_400_000; + const totals = () => ({ credits: 0, usd: 0 as number | null, seconds: 0, runs: new Set() }); + type Totals = ReturnType; + const add = (groups: Map, key: string | null, row: RunCostRow) => { + const g = groups.get(key) ?? totals(); + g.credits += row.credits; + g.usd = g.usd === null || row.usd === null ? null : g.usd + row.usd; + g.seconds += row.seconds; + g.runs.add(row.buildRunId); + groups.set(key, g); + }; + const all = new Map(); + const byJob = new Map(); + const bySize = new Map(); + const byNode = new Map(); + let incompleteRuns = 0; + for (const rec of this.#records) { + if (!rec.run.startedAt || Date.parse(rec.run.startedAt) < since) continue; + for (const row of this.#costRows(scope, rec, now)) { + if (row.resourceType === '_total') { + add(all, null, row); + add(byJob, rec.run.jobKey, row); + if (row.status !== 'complete') incompleteRuns += 1; + } else { + add(bySize, row.warehouseSize || null, row); + add(byNode, row.nodeKey, row); + } + } + } + const total = all.get(null) ?? totals(); + const ranked = (groups: Map) => + [...groups].map(([key, g]) => ({ key, credits: g.credits, usd: g.usd, seconds: g.seconds, runs: g.runs.size })).sort((a, b) => b.credits - a.credits); + return this.#ctx.reply({ + since: new Date(since).toISOString(), + credits: total.credits, + usd: total.runs.size ? total.usd : null, + runs: total.runs.size, + incompleteRuns, + byJob: ranked(byJob), + byWarehouseSize: ranked(bySize), + byNode: ranked(byNode) + }); + } + + /** One node's cost in each build of the window, newest first. */ + listNodeCosts(scope: Scope, nodeKey: string, days: number): Promise { + const now = Date.now(); + const since = now - days * 86_400_000; + const costs: NodeBuildCost[] = []; + for (const rec of this.#records) { + if (!rec.run.startedAt || Date.parse(rec.run.startedAt) < since) continue; + const rows = this.#costRows(scope, rec, now); + const mine = rows.filter((r) => r.nodeKey === nodeKey); + const run = rows.find((r) => r.resourceType === '_total'); + if (!run || !mine.length) continue; + costs.push({ + buildRunId: run.buildRunId, + runId: rec.run.id, + startedAt: rec.run.startedAt, + status: run.status, + credits: mine.reduce((t, r) => t + r.credits, 0), + usd: mine.some((r) => r.usd === null) ? null : mine.reduce((t, r) => t + (r.usd ?? 0), 0), + seconds: mine.reduce((t, r) => t + r.seconds, 0), + queryCount: mine.reduce((t, r) => t + r.queryCount, 0) + }); + } + return this.#ctx.reply(costs.sort((a, b) => b.startedAt.localeCompare(a.startedAt))); + } + + /** + * Cost follows the mock statements at 1 credit an hour on X-Small, 2 on Small, and $3 a credit + * where the target prices credits. + * Only Snowflake estimates query cost, and a run that built nothing (a group run) has none. + */ + #costRows(scope: Scope, rec: RunRecord, now: number): RunCostRow[] { + if (this.#ctx.adapter(scope) !== 'snowflake' || !rec.run.startedAt || !rec.script.nodes.length) return []; + const statements = execution(rec, this.#nodesOf(rec, now), now).statements.filter((st) => st.nodeKey); + const buildRunId = `build-${rec.run.id}`; + // Staging sets no credit price, so it shows credits and the hint to set one. + const priced = this.#ctx.target(scope) !== 'staging'; + const status: RunCostRow['status'] = isActive(rec.run.status) ? 'pending' : 'complete'; + // Mock node keys are bare names, so each resource row carries its name as its node key. + const row = (resourceType: string, resourceName: string, seconds: number, queryCount: number, warehouse: [string, string], credits: number): RunCostRow => ({ + buildRunId, + resourceType, + resourceName, + nodeKey: resourceType === '_total' ? null : resourceName, + warehouseName: warehouse[0], + warehouseSize: warehouse[1], + queryCount, + seconds, + bytesScanned: queryCount * 52_428_800, + credits, + usd: priced ? credits * 3 : null, + status, + statementsExpected: statements.length, + statementsObserved: status === 'pending' ? 0 : statements.length + }); + if (status === 'pending') return [row('_total', buildRunId, 0, 0, ['', ''], 0)]; + const byNode = new Map(); + for (const st of statements) { + const seconds = st.endedAt ? Math.max(0, (Date.parse(st.endedAt) - Date.parse(st.startedAt)) / 1000) : 0; + const node = byNode.get(st.nodeKey) ?? { seconds: 0, queries: 0 }; + node.seconds += seconds; + node.queries += 1; + byNode.set(st.nodeKey, node); + } + const parts = [...byNode].map(([key, node]) => { + const small = /revenue|rollup|segments|status/.test(key); + return row('model', key, node.seconds, node.queries, small ? ['TRANSFORM_WH', 'Small'] : ['LOAD_WH', 'X-Small'], (node.seconds / 3600) * (small ? 2 : 1)); + }); + const sum = (pick: (p: RunCostRow) => number) => parts.reduce((total, p) => total + pick(p), 0); + return [row('_total', buildRunId, sum((p) => p.seconds), statements.length, ['', ''], sum((p) => p.credits)), ...parts]; + } + getStatementSql(scope: Scope, runId: string, statementId: string): Promise { this.#need('view.sql'); const rec = this.#require(runId); diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 018089737f..ea49e2203d 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -286,6 +286,78 @@ export type RunEventPage = { nextCursor: number | null; }; +/** How complete a run's estimated query cost is; `none` until any cost was collected and synced. */ +export type RunCostStatus = 'complete' | 'partial' | 'pending' | 'collection_failed' | 'unavailable' | 'none'; + +/** One latest cost observation: a collection's totals (`resourceType` "_total") or one resource on one warehouse. */ +export type RunCostRow = { + /** The build's own run ID, which groups the rows of one collection. */ + buildRunId: string; + resourceType: string; + resourceName: string; + /** The resource's catalog node key; null on run rows. */ + nodeKey: string | null; + warehouseName: string; + warehouseSize: string; + queryCount: number; + seconds: number; + bytesScanned: number; + credits: number; + /** Null without a configured credit price; the UI never guesses one. */ + usd: number | null; + status: Exclude; + statementsExpected: number; + statementsObserved: number; +}; + +export type RunCostResource = { nodeKey: string; credits: number; usd: number | null; seconds: number; queryCount: number; bytesScanned: number; warehouses: string[] }; +export type RunCostWarehouse = { name: string; size: string; credits: number; usd: number | null; seconds: number; queryCount: number }; + +/** A run's estimated query cost, with its resources and warehouses most expensive first. */ +export type RunCostBreakdown = { + status: RunCostStatus; + credits: number; + usd: number | null; + seconds: number; + queryCount: number; + statementsExpected: number; + statementsObserved: number; + resources: RunCostResource[]; + warehouses: RunCostWarehouse[]; +}; + +/** One group of a cost summary: a job, a warehouse size or a node; `key` is null for builds no server run started. */ +export type CostGroup = { key: string | null; credits: number; usd: number | null; seconds: number; runs: number }; + +/** A target's estimated query cost over recent days, each breakdown most expensive first. */ +export type CostSummary = { + since: string; + credits: number; + /** Null when any run in the window had no configured credit price. */ + usd: number | null; + runs: number; + /** Runs whose cost is still pending or partial, or whose collection failed. */ + incompleteRuns: number; + byJob: CostGroup[]; + byWarehouseSize: CostGroup[]; + /** Every node that ran, keyed by node key. */ + byNode: CostGroup[]; +}; + +/** One node's estimated query cost in one build, across every warehouse it used. */ +export type NodeBuildCost = { + buildRunId: string; + /** The server run that started the build; null for builds from the command line. */ + runId: string | null; + startedAt: string; + /** The collection status of the whole build. */ + status: Exclude; + credits: number; + usd: number | null; + seconds: number; + queryCount: number; +}; + export type RunStatement = { id: string; runId: string; diff --git a/ui/src/lib/operate/cost.test.ts b/ui/src/lib/operate/cost.test.ts new file mode 100644 index 0000000000..b42d193f77 --- /dev/null +++ b/ui/src/lib/operate/cost.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest'; +import type { CatalogNode, RunCostRow } from '$lib/api'; +import { costByFolder, costSummary, formatQueryCost, runCostOf } from './cost'; + +const row = (overrides: Partial): RunCostRow => ({ + nodeKey: `${overrides.resourceType ?? 'model'}:${overrides.resourceName ?? 'orders'}`, + buildRunId: 'b1', + resourceType: 'model', + resourceName: 'orders', + warehouseName: 'LOAD_WH', + warehouseSize: 'X-Small', + queryCount: 1, + seconds: 10, + bytesScanned: 0, + credits: 0.1, + usd: 0.3, + status: 'complete', + statementsExpected: 4, + statementsObserved: 4, + ...overrides +}); +const run = (overrides: Partial): RunCostRow => row({ resourceType: '_total', resourceName: 'b1', nodeKey: null, warehouseName: '', warehouseSize: '', ...overrides }); + +describe("a run's estimated query cost", () => { + it('takes totals from the run row, which also counts queries no node owns', () => { + const cost = runCostOf([run({ credits: 1, usd: 3 }), row({ credits: 0.5, usd: 1.5 }), row({ resourceName: 'customers', credits: 0.25, usd: 0.75 })]); + expect([cost.credits, cost.usd]).toEqual([1, 3]); + expect(cost.resources.map((r) => [r.nodeKey, r.credits])).toEqual([ + ['model:orders', 0.5], + ['model:customers', 0.25] + ]); + }); + + it('folds one node across warehouses and ranks warehouses by cost', () => { + const cost = runCostOf([ + run({ credits: 0.7 }), + row({ credits: 0.1, warehouseName: 'LOAD_WH', warehouseSize: 'X-Small' }), + row({ credits: 0.4, warehouseName: 'TRANSFORM_WH', warehouseSize: 'Small' }), + row({ resourceName: 'customers', credits: 0.2, warehouseName: 'LOAD_WH', warehouseSize: 'X-Small' }) + ]); + expect(cost.resources.map((r) => [r.nodeKey, r.credits, r.warehouses])).toEqual([ + ['model:orders', 0.5, ['LOAD_WH', 'TRANSFORM_WH']], + ['model:customers', 0.2, ['LOAD_WH']] + ]); + expect(cost.warehouses.map((w) => [w.name, w.size])).toEqual([ + ['TRANSFORM_WH', 'Small'], + ['LOAD_WH', 'X-Small'] + ]); + expect(cost.warehouses[1].credits).toBeCloseTo(0.3); + }); + + it("shows a group run's weakest collection and drops dollars when any job run has no price", () => { + const cost = runCostOf([run({ buildRunId: 'b1', status: 'complete' }), run({ buildRunId: 'b2', status: 'partial', usd: null, statementsExpected: 5, statementsObserved: 2 })]); + expect([cost.status, cost.usd, cost.statementsObserved, cost.statementsExpected]).toEqual(['partial', null, 6, 9]); + }); + + it('is none until a collection synced', () => { + expect(runCostOf([]).status).toBe('none'); + }); +}); + +describe('what the cost figure says', () => { + it('never shows missing cost as zero', () => { + expect(costSummary(runCostOf([run({ status: 'pending', credits: 0, usd: 0 })])).text).toBe('Cost pending'); + expect(costSummary(runCostOf([run({ status: 'collection_failed', credits: 0, usd: 0 })])).text).toBe('Cost unavailable'); + }); + + it('names how much of a partial run was counted', () => { + expect(costSummary(runCostOf([run({ status: 'partial', usd: 1.234, statementsExpected: 4, statementsObserved: 3 })]))).toEqual({ text: '$1.23', note: 'Partial: 3 of 4 statements' }); + expect(costSummary(runCostOf([run({ status: 'partial', statementsExpected: 4, statementsObserved: 4 })])).note).toBe('Partial: the warehouse left out some query details'); + }); + + it('shows credits without a configured price and a floor for tiny amounts', () => { + expect(formatQueryCost({ credits: 0.0125, usd: null })).toBe('0.013 credits'); + expect(formatQueryCost({ credits: 0.001, usd: 0.003 })).toBe('< $0.01'); + expect(formatQueryCost({ credits: 0, usd: 0 })).toBe('$0.00'); + }); +}); + +describe('cost by folder', () => { + const node = (key: string, path: string, group: string | null = null) => ({ key, kind: 'model', path, group }) as unknown as CatalogNode; + const nodes = [node('model:stg_orders', 'models/staging/orders/stg_orders.sql'), node('model:stg_items', 'models/staging/stg_items.sql'), node('model:fct_revenue', 'models/finance/fct_revenue.sql', 'marts/finance')]; + + it('sums nodes under their outermost folder, honours declared groups and keeps removed nodes', () => { + const folders = costByFolder( + [ + { key: 'model:fct_revenue', credits: 0.5, usd: 1.5, seconds: 900, runs: 2 }, + { key: 'model:stg_orders', credits: 0.3, usd: 0.9, seconds: 540, runs: 3 }, + { key: 'model:stg_items', credits: 0.4, usd: null, seconds: 720, runs: 3 }, + { key: 'model:dropped', credits: 0.05, usd: 0.15, seconds: 90, runs: 1 } + ], + nodes + ); + expect(folders.map((f) => [f.key, f.nodes, f.usd])).toEqual([ + ['staging', 2, null], + ['marts', 1, 1.5], + ['other', 1, 0.15] + ]); + expect(folders[0].credits).toBeCloseTo(0.7); + }); +}); diff --git a/ui/src/lib/operate/cost.ts b/ui/src/lib/operate/cost.ts new file mode 100644 index 0000000000..d59a6f2507 --- /dev/null +++ b/ui/src/lib/operate/cost.ts @@ -0,0 +1,95 @@ +import type { CatalogNode, CostGroup, RunCostBreakdown, RunCostRow, RunCostStatus } from '$lib/api'; +import { topFolder } from '$lib/lineage/catalog-graph'; + +/** The resource type of the row carrying a collection's totals. */ +export const TOTAL = '_total'; + +/** Shown wherever figures read in credits, because the project set no credit price. */ +export const CREDIT_PRICE_HINT = 'Set usd_per_credit under [cost] in sqlbuild_project.toml to see dollars.'; + +/** Node costs summed per top-level folder, most expensive first; nodes no longer deployed count as `other`. */ +export function costByFolder(byNode: CostGroup[], nodes: CatalogNode[]): { key: string; credits: number; usd: number | null; nodes: number }[] { + const folders = new Map(nodes.map((n) => [n.key, topFolder(n)])); + const totals = new Map(); + for (const g of byNode) { + const key = (g.key && folders.get(g.key)) ?? 'other'; + const t = totals.get(key) ?? { key, credits: 0, usd: 0, nodes: 0 }; + t.credits += g.credits; + t.usd = sumUsd(t.usd, g.usd); + t.nodes += 1; + totals.set(key, t); + } + return [...totals.values()].sort((a, b) => b.credits - a.credits); +} + +/** Least complete first, so a group run shows the weakest of its job runs' collections. */ +const STATUS_RANK: Record = { none: 0, collection_failed: 1, unavailable: 2, pending: 3, partial: 4, complete: 5 }; + +const sumUsd = (a: number | null, b: number | null): number | null => (a == null || b == null ? null : a + b); + +/** Totals from each collection's run row; resources and warehouses folded from the other rows. */ +export function runCostOf(rows: RunCostRow[]): RunCostBreakdown { + const runs = rows.filter((r) => r.resourceType === TOTAL); + const parts = rows.filter((r) => r.resourceType !== TOTAL); + const status: RunCostStatus = runs.length === 0 ? 'none' : runs.map((r) => r.status).reduce((worst, s) => (STATUS_RANK[s] < STATUS_RANK[worst] ? s : worst)); + const resources = new Map(); + const warehouses = new Map(); + for (const r of parts) { + const nodeKey = r.nodeKey ?? r.resourceName; + const resource = resources.get(nodeKey) ?? { nodeKey, credits: 0, usd: 0, seconds: 0, queryCount: 0, bytesScanned: 0, warehouses: [] }; + resource.credits += r.credits; + resource.usd = sumUsd(resource.usd, r.usd); + resource.seconds += r.seconds; + resource.queryCount += r.queryCount; + resource.bytesScanned += r.bytesScanned; + if (r.warehouseName && !resource.warehouses.includes(r.warehouseName)) resource.warehouses.push(r.warehouseName); + resources.set(nodeKey, resource); + const warehouseKey = `${r.warehouseName}|${r.warehouseSize}`; + const warehouse = warehouses.get(warehouseKey) ?? { name: r.warehouseName, size: r.warehouseSize, credits: 0, usd: 0, seconds: 0, queryCount: 0 }; + warehouse.credits += r.credits; + warehouse.usd = sumUsd(warehouse.usd, r.usd); + warehouse.seconds += r.seconds; + warehouse.queryCount += r.queryCount; + warehouses.set(warehouseKey, warehouse); + } + const byCost = (a: T, b: T) => b.credits - a.credits; + return { + status, + credits: runs.reduce((total, r) => total + r.credits, 0), + usd: runs.length === 0 ? null : runs.map((r) => r.usd).reduce(sumUsd), + seconds: runs.reduce((total, r) => total + r.seconds, 0), + queryCount: runs.reduce((total, r) => total + r.queryCount, 0), + statementsExpected: runs.reduce((total, r) => total + r.statementsExpected, 0), + statementsObserved: runs.reduce((total, r) => total + r.statementsObserved, 0), + resources: [...resources.values()].sort(byCost), + warehouses: [...warehouses.values()].sort(byCost) + }; +} + +/** Dollars when the project priced credits; otherwise credits, so no price is ever guessed. */ +export function formatQueryCost(cost: { credits: number; usd: number | null }): string { + if (cost.usd != null) return cost.usd < 0.01 && cost.usd > 0 ? '< $0.01' : `$${cost.usd.toFixed(2)}`; + return `${cost.credits < 0.001 && cost.credits > 0 ? '< 0.001' : cost.credits.toFixed(3)} credits`; +} + +/** What the run's cost figure says; missing data never reads as $0. */ +export function costSummary(cost: RunCostBreakdown): { text: string; note: string | null } { + switch (cost.status) { + case 'none': + return { text: 'No cost yet', note: null }; + case 'pending': + return { text: 'Cost pending', note: 'The warehouse has not reported this run\'s queries yet.' }; + case 'collection_failed': + return { text: 'Cost unavailable', note: 'Reading the warehouse query history failed; a later run retries.' }; + case 'unavailable': + return { text: 'Cost unavailable', note: null }; + case 'partial': + // Every statement can be seen while the warehouse still left out some of their details. + return { + text: formatQueryCost(cost), + note: cost.statementsObserved < cost.statementsExpected ? `Partial: ${cost.statementsObserved} of ${cost.statementsExpected} statements` : 'Partial: the warehouse left out some query details' + }; + case 'complete': + return { text: formatQueryCost(cost), note: null }; + } +} diff --git a/ui/src/routes/+page.svelte b/ui/src/routes/+page.svelte index 32124d2847..1248b434e2 100644 --- a/ui/src/routes/+page.svelte +++ b/ui/src/routes/+page.svelte @@ -23,6 +23,7 @@ import { formatCredits } from '$lib/build-area/labels'; import { nodeChangeLine, summarizeChange } from '$lib/build-area/changes'; import { pagesBeyondTarget } from '$lib/nav'; + import OverviewCost from './overview-cost.svelte'; const overview = createResource((client, scope) => client.overview(scope), ['runs', 'checks', 'node_status', 'snapshot', 'platform_events', 'triggers']); const data = $derived(overview.data ?? null); @@ -221,6 +222,8 @@ {/if} + +
{#if showGroups}
diff --git a/ui/src/routes/catalog/[name]/cost-chart.svelte b/ui/src/routes/catalog/[name]/cost-chart.svelte new file mode 100644 index 0000000000..db135ff2cd --- /dev/null +++ b/ui/src/routes/catalog/[name]/cost-chart.svelte @@ -0,0 +1,54 @@ + + +
+
+

Estimated query cost per build

+ {counted.length} of {builds.length} builds in the last {days} days{#if average} · {formatQueryCost(average)} on average · {formatQueryCost(total)} in total{/if} +
+
+ Cost +
+ {#each builds as c (c.buildRunId)} + + {/each} +
+ Query time +
+ {#each builds as c (c.buildRunId)} + + {/each} +
+
+
+ + diff --git a/ui/src/routes/catalog/[name]/node-view.svelte b/ui/src/routes/catalog/[name]/node-view.svelte index 19f2e3198b..5e8be1770b 100644 --- a/ui/src/routes/catalog/[name]/node-view.svelte +++ b/ui/src/routes/catalog/[name]/node-view.svelte @@ -27,6 +27,7 @@ import { upstreamNote, type Tab } from './utils'; let { name }: { name: string } = $props(); + const COST_DAYS = 30; const res = createResource((client, scope) => client.getCatalogNode(scope, name), ['runs', 'checks', 'snapshot', 'node_status', 'warehouse_sync']); const n = $derived(res.data ?? null); @@ -36,6 +37,7 @@ }); const catalog = createResource((client, scope) => client.listCatalogNodes(scope), ['snapshot', 'runs']); const colLineage = createResource((client, scope) => client.getColumnLineage(scope, name), ['snapshot']); + const costs = createResource((client, scope) => (client.supports('listNodeCosts') ? client.listNodeCosts(scope, name, COST_DAYS) : Promise.resolve([])), ['runs', 'warehouse_sync']); const tabs = $derived.by<[Tab, string][]>(() => { if (!n) return [['overview', 'Overview']]; @@ -161,7 +163,7 @@ {/if} {:else if tab === 'runs'} - + {:else if tab === 'checks'} {:else if tab === 'warehouse' && n.warehouse} diff --git a/ui/src/routes/catalog/[name]/runs-tab.svelte b/ui/src/routes/catalog/[name]/runs-tab.svelte index 47dbedfb61..9f270331f7 100644 --- a/ui/src/routes/catalog/[name]/runs-tab.svelte +++ b/ui/src/routes/catalog/[name]/runs-tab.svelte @@ -1,6 +1,8 @@ +{#if costs.length} + + {#if costs.some((c) => c.usd === null)}

{CREDIT_PRICE_HINT}

{/if} +{/if} + - + {#if costs.length}{/if} {#each node.runs ?? [] as r, i (`${r.runId ?? r.fingerprintRunId}-${i}`)} @@ -29,10 +39,11 @@ + {#if costs.length}{@const c = costOf(r)}{/if} {:else} - + {/each}
OutcomeRunJobRowsDurationStarted
OutcomeRunJobRowsDurationEst. costStarted
{#if r.jobKey}{r.jobKey}{:else}{r.runId ? 'started by hand' : '—'}{/if} {formatNumber(r.rows)} {formatMs(r.durationMs)}{costText(c)}{c?.status === 'partial' ? '*' : ''}
No builds recorded in this location.
No builds recorded in this location.
@@ -48,6 +59,11 @@ .muted { color: var(--muted-foreground); } + .hint { + margin: 0 0 10px; + color: var(--sb-text-faint); + font-size: 12px; + } .detail { margin-top: 3px; font-size: 12px; diff --git a/ui/src/routes/overview-cost.svelte b/ui/src/routes/overview-cost.svelte new file mode 100644 index 0000000000..9c5ecbd450 --- /dev/null +++ b/ui/src/routes/overview-cost.svelte @@ -0,0 +1,111 @@ + + +{#if app.client.supports('getCostSummary') && (cost?.runs || summary.error)} +
+
+
+

Estimated query cost

+ Warehouse time the runs' queries used, not billed warehouse time +
+
+ {#each WINDOWS as w (w)} + + {/each} +
+
+ {#if summary.error && !cost} +

Couldn't load cost: {summary.error}

+ {:else if cost} +
+ {formatQueryCost(cost)} + {plural(cost.runs, 'run')} in the last {days} days{#if cost.incompleteRuns} · {cost.incompleteRuns} still pending or partial{/if} + {#if cost.usd === null}{CREDIT_PRICE_HINT}{/if} +
+
+ {#each columns as c (c.title)} + {@const top = c.rows[0]?.credits || 1} +
+

{c.title}

+ {#each c.rows.slice(0, TOP) as r (r.key)} + + {r.label} + {formatQueryCost(r)} + + {r.detail} + + {/each} + {#if c.rows.length > TOP}and {c.rows.length - TOP} more{/if} +
+ {/each} +
+ {/if} +
+{/if} + + diff --git a/ui/src/routes/runs/[id]/cost-tab.svelte b/ui/src/routes/runs/[id]/cost-tab.svelte new file mode 100644 index 0000000000..274d161032 --- /dev/null +++ b/ui/src/routes/runs/[id]/cost-tab.svelte @@ -0,0 +1,152 @@ + + +
+
+ Estimated query cost + {summary.text} + {#if summary.note}{summary.note}{/if} + {#if counted && cost.usd == null} + {CREDIT_PRICE_HINT} + {/if} +
+ + {#if counted && cost.resources.length} +

By node most expensive first

+ + + + + + {#each shown as r (r.nodeKey)} + + + + + + + + + + {:else} + + {/each} + +
NodeCostShareWarehouse timeQueriesScannedWarehouse
{#if runStatements(r.nodeKey)}Run statements{:else if onnode}{:else}{r.nodeKey}{/if}{formatQueryCost(r)}{share(r.credits)}{formatSpan(r.seconds)}{r.queryCount}{gb(r.bytesScanned)}{r.warehouses.join(', ')}
No cost for the selected nodes.
+ +

By warehouse

+ + + + + + {#each cost.warehouses as w (`${w.name}|${w.size}`)} + + + + + + + + + {/each} + +
WarehouseSizeCostShareWarehouse timeQueries
{w.name}{w.size}{formatQueryCost(w)}{share(w.credits)}{formatSpan(w.seconds)}{w.queryCount}
+

Estimated from the warehouse's query history: each query's share of its warehouse's credits. Idle time and the 60-second minimum are not included.

+ {:else if cost.status === 'none'} +

Cost appears after the location syncs this run's query history from a Snowflake warehouse. Other warehouses show duration only.

+ {/if} +
+ + diff --git a/ui/src/routes/runs/[id]/run-header.svelte b/ui/src/routes/runs/[id]/run-header.svelte index be7b12aef7..d9e7a6cf10 100644 --- a/ui/src/routes/runs/[id]/run-header.svelte +++ b/ui/src/routes/runs/[id]/run-header.svelte @@ -7,6 +7,7 @@ import { app } from '$lib/state/app.svelte'; import { runKindLabel, runOriginLabel, runStatusTone } from '$lib/status'; import RerunButton from './rerun-button.svelte'; + import { costSummary } from './utils'; import type { RunDetailState } from './state.svelte'; let { view, runId }: { view: RunDetailState; runId: string } = $props(); @@ -53,6 +54,11 @@ Started {formatStamp(run.startedAt)} ({formatAgo(run.startedAt, app.now)}) · {view.active ? 'running for ' : 'took '}{formatDuration(run.startedAt, run.endedAt, app.now)} + {#if view.cost && view.cost.status !== 'none'} + {@const summary = costSummary(view.cost)} + · + + {/if} {:else} Queued {formatAgo(run.queuedAt, app.now)} {/if} @@ -104,6 +110,11 @@ {#if view.actionError}{/if}