From ba915265c2a00fb71a99145af786d9d91b4d03d0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:10:50 +0000 Subject: [PATCH 1/3] fix(location): run warehouse syncs on the target's sync or query warehouse A location's warehouse syncs ran under no command group, so they used the connection's warehouse, often the largest, to read metadata and small state tables, waking it for a minimum billed minute each time. The location command now uses a new 'sync' warehouse group: the target's [targets..warehouses] sync, else query, else the connection's own warehouse, as before. The target's build warehouse is used only when the connection names none, so a sync can still run. sqb debug lists the sync warehouse with where it comes from, and the location reports it when a sync session opens. Runs a location executes keep their own command's group. --- .../src/command_connection/constants.rs | 4 +- .../sqlbuild-config/src/project/constants.rs | 2 +- .../_helpers/compiler_enums/constants.rs | 2 +- .../_helpers/contract_records/constants.rs | 1 + .../_helpers/config_validation.rs | 2 +- .../_helpers/command_runtime/warehouses.rs | 76 ++++++++++++++----- .../native_host/warehouse_metadata.rs | 21 ++++- .../sqlbuild/references/docs/cli/debug.md | 1 + .../docs/concepts/adapters/snowflake.md | 15 ++++ .../docs/concepts/project-configuration.md | 2 +- src/sqlbuild/_native.pyi | 3 + .../_helpers/runtime/test_warehouses.py | 42 ++++++++++ .../_helpers/test_target_warehouses.py | 5 ++ website/src/content/docs/docs/cli/debug.mdx | 1 + .../docs/docs/concepts/adapters/snowflake.mdx | 15 ++++ .../docs/concepts/project-configuration.mdx | 2 +- 16 files changed, 167 insertions(+), 27 deletions(-) diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 7a68e32d8a..2d6591fd76 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -23,7 +23,7 @@ pub(crate) const UNSUPPORTED_WAREHOUSE_CODE: &str = "C260"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; -/// The warehouse group each CLI command selects a target's default warehouse from. +/// The warehouse group each CLI command uses; a location's metadata syncs use the cheap `sync` group. pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("audit", Some("build")), ("build", Some("build")), @@ -44,13 +44,13 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("janitor", Some("query")), ("scenario", Some("query")), ("test", Some("query")), + ("location", Some("sync")), ("clean", None), ("compile", None), ("dag", None), ("format", None), ("init", None), ("lineage", None), - ("location", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 99ae5b143c..ad8150d254 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -279,7 +279,7 @@ pub(crate) const EVENT_EXPORT_SEVERITIES: [&str; 4] = ["debug", "info", "warning /// `TIME_TRAVEL_RETENTION_MATERIALIZATIONS` in declared order. pub(crate) const RETENTION_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; pub(crate) const MATERIALIZATION_DEFAULT_KEYS: [&str; 2] = ["time_travel_retention", "table_type"]; -pub(crate) const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"]; +pub(crate) const WAREHOUSE_GROUPS: [&str; 3] = ["build", "query", "sync"]; pub(crate) const SNAPSHOT_FULL_REFRESH_POLICIES: [&str; 3] = ["allow", "deny", "require_confirmation"]; pub(crate) const SNAPSHOT_SCHEMA_CHANGE_POLICIES: [&str; 3] = diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index b9d86c1362..63f2aa3e7b 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -342,7 +342,7 @@ const E_EVENT_EXPORT_SEVERITY: EnumSpec = EnumSpec { const E_WAREHOUSE_GROUP: EnumSpec = EnumSpec { name: "WarehouseGroup", doc: "Command group that selects a target's default warehouse.", - members: &[("BUILD", "build"), ("QUERY", "query")], + members: &[("BUILD", "build"), ("QUERY", "query"), ("SYNC", "sync")], }; const E_SOURCE_WRITE_STRATEGY: EnumSpec = EnumSpec { name: "SourceWriteStrategy", diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index d3fdf60b51..c041a0efc2 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -102,6 +102,7 @@ const R_TARGET_WAREHOUSES_CONFIG: RecordSpec = RecordSpec { fields: &[ FieldSpec::new("build", FieldDefault::None), FieldSpec::new("query", FieldDefault::None), + FieldSpec::new("sync", FieldDefault::None), ], }; const R_TARGET_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs index 19a5cafb28..b41b92014b 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs @@ -19,7 +19,7 @@ use sqlbuild_python_base::records::types::ErrorId; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; /// `WarehouseGroup` members in declaration order. -const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"]; +const WAREHOUSE_GROUPS: [&str; 3] = ["build", "query", "sync"]; fn project_config_error(py: Python<'_>, message: &str, help: Option<&str>) -> PyErr { new_error(py, ErrorId::ProjectConfig, message, (None, help)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 20515e7b7a..52ccd14ab5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -34,9 +34,17 @@ use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_ use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; use sqlbuild_python_front::model_config::main::config::expanded_template_data::expanded_template_data; -/// The `WarehouseGroup` whose warehouse is a target's `warehouses.build`; any other reads `query`. const BUILD_GROUP: &str = "build"; const QUERY_GROUP: &str = "query"; +const SYNC_GROUP: &str = "sync"; +/// The target warehouses each group reads, in order, before the connection's own warehouse. +const GROUP_WAREHOUSES: [(&str, &[&str]); 3] = [ + (BUILD_GROUP, &[BUILD_GROUP]), + (QUERY_GROUP, &[QUERY_GROUP]), + (SYNC_GROUP, &[SYNC_GROUP, QUERY_GROUP]), +]; +/// The target warehouse a sync falls back to only when the connection names none, so it can still run. +const SYNC_LAST_RESORT: &str = BUILD_GROUP; const RESOLVE_OPTIONS: (&[&str], &[&str]) = ( &[ @@ -115,15 +123,10 @@ fn resolved_warehouse<'py>( ) } -fn group_value(target: &Bound<'_, PyAny>, group_name: &str) -> PyResult> { +fn group_value(target: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { if target.is_none() { return Ok(None); } - let attribute: &str = if group_name == BUILD_GROUP { - BUILD_GROUP - } else { - QUERY_GROUP - }; optional_text(&native_field( &native_field(target, "warehouses")?, attribute, @@ -135,8 +138,27 @@ pub(crate) fn find_authored( inputs: WarehouseInputs<'_, '_>, group: &Bound<'_, PyAny>, ) -> PyResult> { - let group_name: String = native_field(group, "value")?.extract()?; - authored_group(inputs, &group_name) + authored_group(inputs, &group_name(group)?) +} + +/// The name of a command group, such as `sync`. +fn group_name(group: &Bound<'_, PyAny>) -> PyResult { + native_field(group, "value")?.extract() +} + +/// The target warehouse `group` uses before the connection's; a sync uses `build` only when the connection names none. +fn group_authored( + inputs: WarehouseInputs<'_, '_>, + group: &Bound<'_, PyAny>, + connection_config: &Bound<'_, PyAny>, +) -> PyResult> { + if let Some(found) = find_authored(inputs, group)? { + return Ok(Some(found)); + } + if group_name(group)? == SYNC_GROUP && connection_warehouse(connection_config)?.is_none() { + return authored_group(inputs, SYNC_LAST_RESORT); + } + Ok(None) } /// The active target's authored, unexpanded warehouse for the group named `group_name`. @@ -163,11 +185,20 @@ fn authored_group<'py>( let project_target: Bound<'py, PyAny> = native_field(&project_config, "targets")?.call_method1("get", (&target_name,))?; let target_name: String = target_name.extract()?; - Ok(authored_warehouse( - (&target_name, group_name), - group_value(&local_target, group_name)?, - group_value(&project_target, group_name)?, - )) + let attributes: &[&str] = GROUP_WAREHOUSES + .iter() + .find(|(group, _)| *group == group_name) + .map_or(&[QUERY_GROUP], |(_, attributes)| attributes); + for attribute in attributes { + if let Some(found) = authored_warehouse( + (&target_name, attribute), + group_value(&local_target, attribute)?, + group_value(&project_target, attribute)?, + ) { + return Ok(Some(found)); + } + } + Ok(None) } /// The group warehouse with `${...}` templates expanded, checked as an identifier. @@ -236,7 +267,7 @@ pub(crate) fn resolve_warehouse( } if let Some(inputs) = inputs && !group.is_none() - && let Some(found) = find_authored(inputs, group)? + && let Some(found) = group_authored(inputs, group, connection_config)? { let warehouse: String = expand_authored(&found, inputs, cli_vars)?; return Ok((Some(warehouse), found.source())); @@ -247,14 +278,14 @@ pub(crate) fn resolve_warehouse( }) } -/// Every warehouse the target configures: its `build` and `query` group warehouses and its connection warehouse. +/// Every warehouse the target configures: its `build`, `query` and `sync` group warehouses and its connection warehouse. pub(crate) fn configured_warehouses( inputs: WarehouseInputs<'_, '_>, connection_config: &Bound<'_, PyAny>, ) -> PyResult> { let none: Bound<'_, PyAny> = none(connection_config.py()); let mut names: std::collections::BTreeSet = std::collections::BTreeSet::new(); - for group_name in [BUILD_GROUP, QUERY_GROUP] { + for group_name in [BUILD_GROUP, QUERY_GROUP, SYNC_GROUP] { if let Some(authored) = authored_group(inputs, group_name)? { names.insert(expand_authored(&authored, inputs, &none)?); } @@ -328,7 +359,9 @@ pub(crate) fn apply_warehouse<'py>( let py: Python<'py> = config.py(); let scope: CommandScope<'py> = command_scope(py)?; let authored: bool = match inputs { - Some(inputs) if !scope.group.is_none() => find_authored(inputs, &scope.group)?.is_some(), + Some(inputs) if !scope.group.is_none() => { + group_authored(inputs, &scope.group, config)?.is_some() + } _ => false, }; if scope.cli_warehouse.is_none() && !authored { @@ -398,7 +431,12 @@ pub(crate) fn describe_connection_warehouse<'py>( let found: Option = if scope.group.is_none() { None } else { - find_authored(inputs, &scope.group)? + match find_authored(inputs, &scope.group)? { + None if group_name(&scope.group)? == SYNC_GROUP => { + authored_group(inputs, SYNC_LAST_RESORT)? + } + found => found, + } }; if let Some(found) = found { let expands_to: bool = match expand_authored(&found, inputs, &scope.cli_vars) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index a0c0027b74..b39e4a82a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; use serde_json::{Value, json}; use sqlbuild_analysis::fingerprint_storage::constants::{COLUMN_TIMESTAMP, FINGERPRINT_COLUMNS}; use sqlbuild_config::command_connection::constants::{ - DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, + CONNECTION_WAREHOUSE_KEY, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, @@ -42,8 +42,10 @@ use crate::bindings::_helpers::query_command::project::project_adapter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::constants::HISTORY_STREAM_PAGES; +use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; @@ -92,6 +94,22 @@ pub(crate) struct HistoryStream<'py> { since: Option>, } +/// Say which warehouse a sync session runs on, so a deployment sees it is not the largest one. +fn report_sync_warehouse(py: Python<'_>, connection_config: &Bound<'_, PyAny>) -> PyResult<()> { + let warehouse: Bound<'_, PyAny> = mapping_get(connection_config, CONNECTION_WAREHOUSE_KEY)?; + if warehouse.is_none() { + return Ok(()); + } + write_console( + py, + "stderr", + &format!( + "Warehouse sync session opened on warehouse {}.\n", + warehouse.str()? + ), + ) +} + pub(crate) fn open_history_session<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, @@ -111,6 +129,7 @@ pub(crate) fn open_history_session<'py>( (&py.None().into_bound(py), true), true, )?; + report_sync_warehouse(py, &connection_config)?; let adapter_name: String = adapter.getattr("adapter_name")?.extract()?; let local_path_key = LOCAL_DATABASE_PATH_KEYS .iter() diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md index fff88cb037..5598592a37 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md @@ -71,6 +71,7 @@ Configuration: schema: DEV [OK resolved] build warehouse: BUILD_WH [OK sqlbuild_project.toml [targets.dev.warehouses] build] query warehouse: TRANSFORM_WH [OK connection warehouse] + sync warehouse: TRANSFORM_WH [OK connection warehouse] Providers: providers: none discovered [OK none discovered] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 08b57421cf..dcedd89b3c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -84,6 +84,7 @@ query = "ADHOC_WH" |-------|----------| | `build` | `audit`, `build`, `check`, `clone`, `dbt` (SQLBuild's own work), `load`, `plan`, `seed` | | `query` | `contract`, `cost`, `debug`, `diff`, `freshness`, `janitor`, `query`, `scenario`, `test` | +| `sync` | `location`: a code location's warehouse syncs, which read metadata and SQLBuild's small state tables | Each command uses the first of these that is set: @@ -92,6 +93,20 @@ Each command uses the first of these that is set: 3. The connection's `warehouse` 4. The Snowflake user's default warehouse +A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the +connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read +metadata. Only when the connection sets no `warehouse` does a sync use the `build` warehouse. Set +`sync` when the syncs need their own warehouse: + +```toml +[targets.dev.warehouses] +build = "BUILD_WH" +query = "ADHOC_WH" +sync = "TINY_WH" +``` + +Runs that a location executes, such as `sqb build`, use their own command's group. + Every connection a command opens uses the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index ed51b47f92..63c1f46108 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -185,7 +185,7 @@ A target is a named build context - the database and schema you build into, plus | `defer_sources_to` | Target name to read managed source data from (see [Loaders](python-nodes/loaders.md#source-deferral)) | | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny`. See [Missing migration origins](models/migrations.md#missing-migration-origins) | -| `warehouses` | Snowflake warehouse per command group, `build` and `query`. See [Command-group warehouses](adapters/snowflake.md#command-group-warehouses) | +| `warehouses` | Snowflake warehouse per command group: `build`, `query` and `sync` (a location's warehouse syncs). See [Command-group warehouses](adapters/snowflake.md#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index de71fe3845..54daf3725c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -11742,6 +11742,7 @@ class WarehouseGroup(StrEnum): BUILD = "build" QUERY = "query" + SYNC = "sync" class SourceWriteStrategy(StrEnum): APPEND = "append" @@ -11954,10 +11955,12 @@ class TargetWarehousesConfig(NativeRecord): build: str | None query: str | None + sync: str | None def __init__( self, build: str | None = ..., query: str | None = ..., + sync: str | None = ..., ) -> None: ... class TargetConfig(NativeRecord): diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index bf38d9db07..3f09263a98 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -77,6 +77,43 @@ expected_warehouse="BUILD_WH", expected_source="sqlbuild_project.toml [targets.dev.warehouses] build", ), + CommandWarehouseResolutionTestCase( + description="a location syncs on the query warehouse, not the connection's", + command="location", + project_warehouses=_PROJECT_GROUPS, + expected_warehouse="ADHOC_WH", + expected_source="sqlbuild_project.toml [targets.dev.warehouses] query", + ), + CommandWarehouseResolutionTestCase( + description="a location syncs on the sync warehouse when the target sets one", + command="location", + project_warehouses=TargetWarehousesConfig( + build="BUILD_WH", query="ADHOC_WH", sync="SYNC_WH" + ), + expected_warehouse="SYNC_WH", + expected_source="sqlbuild_project.toml [targets.dev.warehouses] sync", + ), + CommandWarehouseResolutionTestCase( + description="a location with only a build warehouse syncs on the connection's, not the build one", + command="location", + project_warehouses=TargetWarehousesConfig(build="XL_WH"), + expected_warehouse="ANALYTICS_WH", + expected_source="connection warehouse", + ), + CommandWarehouseResolutionTestCase( + description="a location with only a build warehouse and no connection warehouse syncs on the build one", + command="location", + project_warehouses=TargetWarehousesConfig(build="XL_WH"), + connection={"account": "example-account"}, + expected_warehouse="XL_WH", + expected_source="sqlbuild_project.toml [targets.dev.warehouses] build", + ), + CommandWarehouseResolutionTestCase( + description="a location whose target sets no warehouse keeps the connection's", + command="location", + expected_warehouse="ANALYTICS_WH", + expected_source="connection warehouse", + ), CommandWarehouseResolutionTestCase( description="cli warehouse overrides the group", command="build", @@ -361,6 +398,11 @@ def test_given_command_without_group_when_parsing_warehouse_flag_then_it_is_reje command=CliCommand.BUILD, expected_group=WarehouseGroup.BUILD, ), + CommandWarehouseGroupTestCase( + description="a location's syncs read metadata and small state tables", + command=CliCommand.LOCATION, + expected_group=WarehouseGroup.SYNC, + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index 413b040910..9b62cb5992 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -63,6 +63,11 @@ class DuckDbPlusAdapter(DuckDbAdapter): + '\n\n[targets.dev.warehouses]\nbuild = "BUILD_WH"\nquery = "ADHOC_WH"\n', expected_warehouses=TargetWarehousesConfig(build="BUILD_WH", query="ADHOC_WH"), ), + TargetWarehousesConfigTestCase( + description="project target sets a sync warehouse for its location", + contents=_SNOWFLAKE_PROJECT + '\n\n[targets.dev.warehouses]\nsync = "SYNC_WH"\n', + expected_warehouses=TargetWarehousesConfig(sync="SYNC_WH"), + ), TargetWarehousesConfigTestCase( description="target without warehouses keeps both groups unset", contents=_SNOWFLAKE_PROJECT, diff --git a/website/src/content/docs/docs/cli/debug.mdx b/website/src/content/docs/docs/cli/debug.mdx index d726fc0b05..bb9022fabc 100644 --- a/website/src/content/docs/docs/cli/debug.mdx +++ b/website/src/content/docs/docs/cli/debug.mdx @@ -70,6 +70,7 @@ Configuration: schema: DEV [OK resolved] build warehouse: BUILD_WH [OK sqlbuild_project.toml [targets.dev.warehouses] build] query warehouse: TRANSFORM_WH [OK connection warehouse] + sync warehouse: TRANSFORM_WH [OK connection warehouse] Providers: providers: none discovered [OK none discovered] diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index bb910975cb..c0c1319e5b 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -69,6 +69,7 @@ query = "ADHOC_WH" |-------|----------| | `build` | `audit`, `build`, `check`, `clone`, `dbt` (SQLBuild's own work), `load`, `plan`, `seed` | | `query` | `contract`, `cost`, `debug`, `diff`, `freshness`, `janitor`, `query`, `scenario`, `test` | +| `sync` | `location`: a code location's warehouse syncs, which read metadata and SQLBuild's small state tables | Each command uses the first of these that is set: @@ -77,6 +78,20 @@ Each command uses the first of these that is set: 3. The connection's `warehouse` 4. The Snowflake user's default warehouse +A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the +connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read +metadata. Only when the connection sets no `warehouse` does a sync use the `build` warehouse. Set +`sync` when the syncs need their own warehouse: + +```toml +[targets.dev.warehouses] +build = "BUILD_WH" +query = "ADHOC_WH" +sync = "TINY_WH" +``` + +Runs that a location executes, such as `sqb build`, use their own command's group. + Every connection a command opens uses the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 2da808b91d..908c9fd342 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -159,7 +159,7 @@ A target is a named build context - the database and schema you build into, plus | `defer_sources_to` | Target name to read managed source data from (see [Loaders](/docs/concepts/python-nodes/loaders#source-deferral)) | | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny`. See [Missing migration origins](/docs/concepts/models/migrations#missing-migration-origins) | -| `warehouses` | Snowflake warehouse per command group, `build` and `query`. See [Command-group warehouses](/docs/concepts/adapters/snowflake#command-group-warehouses) | +| `warehouses` | Snowflake warehouse per command group: `build`, `query` and `sync` (a location's warehouse syncs). See [Command-group warehouses](/docs/concepts/adapters/snowflake#command-group-warehouses) | | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | From 5aee732d5becc83aa025cdd496c39ab408113c97 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:25:57 +0000 Subject: [PATCH 2/3] refactor(cost): read and build cost run records in the resource costs edge The conversions between CostRunRecord objects and plain cost runs call into Python, so they move from the bindings crate to sqlbuild-python-front's resource_costs area, a configured Python edge, next to the resource costs table they feed. Behaviour is unchanged. The bindings crate drops below its code-line cap again, and the two pending edge budgets for the old file go away. --- .../_helpers/cost_run_records.rs | 134 ++++++++++ .../src/resource_costs/_helpers/mod.rs | 1 + .../src/resource_costs/main/artifact_error.rs | 16 ++ .../src/resource_costs/main/cost_run.rs | 25 ++ .../src/resource_costs/main/cost_runs.rs | 14 ++ .../src/resource_costs/main/mod.rs | 6 + .../src/resource_costs/main/record_object.rs | 31 +++ .../src/resource_costs/main/record_tuple.rs | 15 ++ .../src/resource_costs/main/summary.rs | 41 +++ .../_helpers/cost_report/degradation.rs | 2 +- .../_helpers/cost_report/functions.rs | 7 +- .../src/bindings/_helpers/cost_report/mod.rs | 1 - .../_helpers/cost_report/publication.rs | 2 +- .../bindings/_helpers/cost_report/records.rs | 236 ------------------ .../bindings/_helpers/cost_report/store.rs | 2 +- fensu.toml | 2 - 16 files changed, 290 insertions(+), 245 deletions(-) create mode 100644 crates/sqlbuild-python-front/src/resource_costs/_helpers/cost_run_records.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/artifact_error.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/record_object.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/record_tuple.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/summary.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs diff --git a/crates/sqlbuild-python-front/src/resource_costs/_helpers/cost_run_records.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/cost_run_records.rs new file mode 100644 index 0000000000..ca95cb231f --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/cost_run_records.rs @@ -0,0 +1,134 @@ +//! Plain cost runs read from `CostRunRecord` objects, and their fields built back into Python values. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_core::cost_report::models::{CostResource, CostSummary}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +pub(crate) fn text(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + native_field(value, field)?.extract() +} + +pub(crate) fn optional_text(value: &Bound<'_, PyAny>, field: &str) -> PyResult> { + native_field(value, field)?.extract() +} + +/// `str(value.)`, the text a `Decimal` field is stored as. +pub(crate) fn decimal(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + Ok(native_field(value, field)?.str()?.to_string()) +} + +pub(crate) fn integer(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + native_field(value, field)?.extract() +} + +pub(crate) fn timestamp(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + native_field(value, field)? + .call_method0("isoformat")? + .extract() +} + +pub(crate) fn resource(value: &Bound<'_, PyAny>) -> PyResult { + Ok(CostResource { + resource_type: text(value, "resource_type")?, + resource_name: text(value, "resource_name")?, + warehouse_name: text(value, "warehouse_name")?, + warehouse_size: text(value, "warehouse_size")?, + warehouse_type: optional_text(value, "warehouse_type")?, + cluster_number: native_field(value, "cluster_number")?.extract()?, + query_count: integer(value, "query_count")?, + attributed_seconds: decimal(value, "attributed_seconds")?, + bytes_scanned: integer(value, "bytes_scanned")?, + estimated_compute_credits: decimal(value, "estimated_compute_credits")?, + estimated_usd: decimal(value, "estimated_usd")?, + }) +} + +/// Python constructors of the record fields. +pub(crate) struct Builders<'py> { + pub(crate) decimal: Bound<'py, PyAny>, + pub(crate) timestamp: Bound<'py, PyAny>, +} + +impl<'py> Builders<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + decimal: imported(py, "decimal", "Decimal")?, + timestamp: imported(py, "datetime", "datetime")?.getattr("fromisoformat")?, + }) + } + + pub(crate) fn decimal(&self, text: &str) -> PyResult> { + self.decimal.call1((text,)) + } +} + +pub(crate) fn resource_object<'py>( + builders: &Builders<'py>, + resource: &CostResource, +) -> PyResult> { + let py: Python<'py> = builders.decimal.py(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("resource_type", &resource.resource_type)?; + fields.set_item("resource_name", &resource.resource_name)?; + fields.set_item("warehouse_name", &resource.warehouse_name)?; + fields.set_item("warehouse_size", &resource.warehouse_size)?; + fields.set_item("warehouse_type", &resource.warehouse_type)?; + fields.set_item("cluster_number", resource.cluster_number)?; + fields.set_item("query_count", resource.query_count)?; + fields.set_item( + "attributed_seconds", + builders.decimal(&resource.attributed_seconds)?, + )?; + fields.set_item("bytes_scanned", resource.bytes_scanned)?; + fields.set_item( + "estimated_compute_credits", + builders.decimal(&resource.estimated_compute_credits)?, + )?; + fields.set_item("estimated_usd", builders.decimal(&resource.estimated_usd)?)?; + new_record(py, RecordId::ResourceCost, (), Some(&fields)) +} + +pub(crate) fn summary_object<'py>( + builders: &Builders<'py>, + cost: &CostSummary, +) -> PyResult> { + let py: Python<'py> = builders.decimal.py(); + let mut resources: Vec> = Vec::with_capacity(cost.resources.len()); + for resource in &cost.resources { + resources.push(resource_object(builders, resource)?); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item( + "status", + host_entry(py, ExecutorHost::CostStatus)?.call1((cost.status.as_str(),))?, + )?; + fields.set_item("usd_per_credit", builders.decimal(&cost.usd_per_credit)?)?; + fields.set_item("rate_source", &cost.rate_source)?; + fields.set_item("resources", PyTuple::new(py, resources)?)?; + fields.set_item("query_count", cost.query_count)?; + fields.set_item( + "attributed_seconds", + builders.decimal(&cost.attributed_seconds)?, + )?; + fields.set_item("bytes_scanned", cost.bytes_scanned)?; + fields.set_item( + "estimated_compute_credits", + builders.decimal(&cost.estimated_compute_credits)?, + )?; + fields.set_item("estimated_usd", builders.decimal(&cost.estimated_usd)?)?; + fields.set_item("expected_statement_count", cost.expected_statement_count)?; + fields.set_item("observed_statement_count", cost.observed_statement_count)?; + fields.set_item("missing_statement_count", cost.missing_statement_count)?; + fields.set_item("skipped_statement_count", cost.skipped_statement_count)?; + fields.set_item("source", &cost.source)?; + fields.set_item("method", &cost.method)?; + fields.set_item("limitations", PyTuple::new(py, &cost.limitations)?)?; + fields.set_item("message", &cost.message)?; + host_entry(py, ExecutorHost::RunCostSummary)?.call((), Some(&fields)) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs index 7d7aae7125..4367bf553f 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod cost_run_records; pub(crate) mod resource_costs_state; diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/artifact_error.rs b/crates/sqlbuild-python-front/src/resource_costs/main/artifact_error.rs new file mode 100644 index 0000000000..2017c6bfc2 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/artifact_error.rs @@ -0,0 +1,16 @@ +//! The `CostArtifactError` Python raised for a cost artifact. + +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; +use sqlbuild_core::cost_report::errors::CostArtifactError; +use sqlbuild_python_base::records::main::registry::native_attr::native_attr; +use sqlbuild_python_base::records::types::NativeAttr; + +/// The `CostArtifactError` Python raised for the same artifact. +pub fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr { + match native_attr(py, NativeAttr::CostArtifactError) + .and_then(|class| class.call1((error.message.as_str(),))) + { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs b/crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs new file mode 100644 index 0000000000..239ae71867 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs @@ -0,0 +1,25 @@ +//! A `CostRunRecord` as a plain run. + +use crate::resource_costs::_helpers::cost_run_records::integer; +use crate::resource_costs::_helpers::cost_run_records::optional_text; +use crate::resource_costs::_helpers::cost_run_records::text; +use crate::resource_costs::_helpers::cost_run_records::timestamp; +use crate::resource_costs::main::summary::summary; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_report::models::CostRun; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A `CostRunRecord` as a plain run. +pub fn cost_run(record: &Bound<'_, PyAny>) -> PyResult { + Ok(CostRun { + run_id: text(record, "run_id")?, + adapter_name: text(record, "adapter_name")?, + target_name: optional_text(record, "target_name")?, + build_status: text(record, "build_status")?, + started_at: timestamp(record, "started_at")?, + completed_at: timestamp(record, "completed_at")?, + cost: summary(&native_field(record, "cost")?)?, + version: integer(record, "version")?, + had_executable_work: native_field(record, "had_executable_work")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs b/crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs new file mode 100644 index 0000000000..7ba02d5fee --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs @@ -0,0 +1,14 @@ +//! Every `CostRunRecord` of an iterable as a plain run. + +use crate::resource_costs::main::cost_run::cost_run; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_report::models::CostRun; + +/// Every record of an iterable as a plain run. +pub fn cost_runs(records: &Bound<'_, PyAny>) -> PyResult> { + let mut runs: Vec = Vec::new(); + for record in records.try_iter()? { + runs.push(cost_run(&record?)?); + } + Ok(runs) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs b/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs index 14fe99725c..5da7f95174 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs @@ -1 +1,7 @@ +pub mod artifact_error; +pub mod cost_run; +pub mod cost_runs; +pub mod record_object; +pub mod record_tuple; pub mod register_resource_costs_state; +pub mod summary; diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/record_object.rs b/crates/sqlbuild-python-front/src/resource_costs/main/record_object.rs new file mode 100644 index 0000000000..f991d0996d --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/record_object.rs @@ -0,0 +1,31 @@ +//! A plain run as the `CostRunRecord` it was read from. + +use crate::resource_costs::_helpers::cost_run_records::Builders; +use crate::resource_costs::_helpers::cost_run_records::summary_object; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::cost_report::models::CostRun; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// A plain run as the `CostRunRecord` it was read from. +pub fn record_object<'py>(py: Python<'py>, run: &CostRun) -> PyResult> { + let builders: Builders<'py> = Builders::new(py)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("version", run.version)?; + fields.set_item("run_id", &run.run_id)?; + fields.set_item("adapter_name", &run.adapter_name)?; + fields.set_item("target_name", &run.target_name)?; + fields.set_item("build_status", &run.build_status)?; + fields.set_item( + "started_at", + builders.timestamp.call1((run.started_at.as_str(),))?, + )?; + fields.set_item( + "completed_at", + builders.timestamp.call1((run.completed_at.as_str(),))?, + )?; + fields.set_item("cost", summary_object(&builders, &run.cost)?)?; + fields.set_item("had_executable_work", run.had_executable_work)?; + new_record(py, RecordId::CostRunRecord, (), Some(&fields)) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/record_tuple.rs b/crates/sqlbuild-python-front/src/resource_costs/main/record_tuple.rs new file mode 100644 index 0000000000..1e1baa2ce7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/record_tuple.rs @@ -0,0 +1,15 @@ +//! Records of several plain runs, as a tuple. + +use crate::resource_costs::main::record_object::record_object; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_core::cost_report::models::CostRun; + +/// Records of several plain runs, as a tuple. +pub fn record_tuple<'py>(py: Python<'py>, runs: &[CostRun]) -> PyResult> { + let mut records: Vec> = Vec::with_capacity(runs.len()); + for run in runs { + records.push(record_object(py, run)?); + } + PyTuple::new(py, records) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/summary.rs b/crates/sqlbuild-python-front/src/resource_costs/main/summary.rs new file mode 100644 index 0000000000..84dbb5cf30 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/summary.rs @@ -0,0 +1,41 @@ +//! A `RunCostSummary` as a plain cost summary. + +use crate::resource_costs::_helpers::cost_run_records::decimal; +use crate::resource_costs::_helpers::cost_run_records::integer; +use crate::resource_costs::_helpers::cost_run_records::optional_text; +use crate::resource_costs::_helpers::cost_run_records::resource; +use crate::resource_costs::_helpers::cost_run_records::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_report::models::{CostResource, CostSummary}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A `RunCostSummary` as a plain cost summary. +pub fn summary(value: &Bound<'_, PyAny>) -> PyResult { + let mut resources: Vec = Vec::new(); + for item in native_field(value, "resources")?.try_iter()? { + resources.push(resource(&item?)?); + } + let mut limitations: Vec = Vec::new(); + for item in native_field(value, "limitations")?.try_iter()? { + limitations.push(item?.extract()?); + } + Ok(CostSummary { + status: native_field(value, "status")?.getattr("value")?.extract()?, + usd_per_credit: decimal(value, "usd_per_credit")?, + rate_source: text(value, "rate_source")?, + resources, + query_count: integer(value, "query_count")?, + attributed_seconds: decimal(value, "attributed_seconds")?, + bytes_scanned: integer(value, "bytes_scanned")?, + estimated_compute_credits: decimal(value, "estimated_compute_credits")?, + estimated_usd: decimal(value, "estimated_usd")?, + expected_statement_count: integer(value, "expected_statement_count")?, + observed_statement_count: integer(value, "observed_statement_count")?, + missing_statement_count: integer(value, "missing_statement_count")?, + skipped_statement_count: integer(value, "skipped_statement_count")?, + source: text(value, "source")?, + method: text(value, "method")?, + limitations, + message: optional_text(value, "message")?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 39ea17272d..6949d5c776 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -14,7 +14,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::cost_report::attribution::report_cost_status; use crate::bindings::_helpers::cost_report::publication::publish_resource_costs; -use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -22,6 +21,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; +use sqlbuild_python_front::resource_costs::main::cost_run::cost_run; const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs index f6145f6d98..c36ac70df1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs @@ -13,11 +13,12 @@ use sqlbuild_core::cost_report::main::format_cost_history::format_cost_history a use sqlbuild_core::cost_report::main::select_cost_run::select_cost_run; use sqlbuild_core::cost_report::models::{BreakdownOptions, CostRun}; -use crate::bindings::_helpers::cost_report::records::{ - cost_run, cost_runs, record_object, record_tuple, -}; use crate::bindings::_helpers::cost_report::store; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_front::resource_costs::main::cost_run::cost_run; +use sqlbuild_python_front::resource_costs::main::cost_runs::cost_runs; +use sqlbuild_python_front::resource_costs::main::record_object::record_object; +use sqlbuild_python_front::resource_costs::main::record_tuple::record_tuple; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; const BREAKDOWN_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index b7f6602189..81fa2bd091 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -3,6 +3,5 @@ pub(crate) mod degradation; pub(crate) mod exception_types; pub(crate) mod functions; pub(crate) mod publication; -pub(crate) mod records; pub(crate) mod registration; pub(crate) mod store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs index 34a11bf9c2..6126b5b574 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs @@ -11,8 +11,8 @@ use sqlbuild_state::resource_costs::constants::COLLECTION_FAILED_STATUS; use sqlbuild_state::resource_costs::main::refresh_candidates::refresh_candidates; use sqlbuild_state::resource_costs::models::RunCostAttempt; -use crate::bindings::_helpers::cost_report::records::summary; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::resource_costs::main::summary::summary; use sqlbuild_python_front::resource_costs::models::CostRunIdentity; use sqlbuild_python_front::resource_costs::models::ResourceCostStore; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs deleted file mode 100644 index 647bf3805b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! `CostRunRecord` objects read into plain cost runs, and plain runs built back into records. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use sqlbuild_core::cost_report::errors::CostArtifactError; -use sqlbuild_core::cost_report::models::{CostResource, CostRun, CostSummary}; -use sqlbuild_host::entries::models::ExecutorHost; - -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::registry::native_attr::native_attr; -use sqlbuild_python_base::records::types::NativeAttr; -use sqlbuild_python_base::records::types::RecordId; - -/// The `CostArtifactError` Python raised for the same artifact. -pub(crate) fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr { - match native_attr(py, NativeAttr::CostArtifactError) - .and_then(|class| class.call1((error.message.as_str(),))) - { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - -fn text(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - native_field(value, field)?.extract() -} - -fn optional_text(value: &Bound<'_, PyAny>, field: &str) -> PyResult> { - native_field(value, field)?.extract() -} - -/// `str(value.)`, the text a `Decimal` field is stored as. -fn decimal(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - Ok(native_field(value, field)?.str()?.to_string()) -} - -fn integer(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - native_field(value, field)?.extract() -} - -fn timestamp(value: &Bound<'_, PyAny>, field: &str) -> PyResult { - native_field(value, field)? - .call_method0("isoformat")? - .extract() -} - -fn resource(value: &Bound<'_, PyAny>) -> PyResult { - Ok(CostResource { - resource_type: text(value, "resource_type")?, - resource_name: text(value, "resource_name")?, - warehouse_name: text(value, "warehouse_name")?, - warehouse_size: text(value, "warehouse_size")?, - warehouse_type: optional_text(value, "warehouse_type")?, - cluster_number: native_field(value, "cluster_number")?.extract()?, - query_count: integer(value, "query_count")?, - attributed_seconds: decimal(value, "attributed_seconds")?, - bytes_scanned: integer(value, "bytes_scanned")?, - estimated_compute_credits: decimal(value, "estimated_compute_credits")?, - estimated_usd: decimal(value, "estimated_usd")?, - }) -} - -pub(crate) fn summary(value: &Bound<'_, PyAny>) -> PyResult { - let mut resources: Vec = Vec::new(); - for item in native_field(value, "resources")?.try_iter()? { - resources.push(resource(&item?)?); - } - let mut limitations: Vec = Vec::new(); - for item in native_field(value, "limitations")?.try_iter()? { - limitations.push(item?.extract()?); - } - Ok(CostSummary { - status: native_field(value, "status")?.getattr("value")?.extract()?, - usd_per_credit: decimal(value, "usd_per_credit")?, - rate_source: text(value, "rate_source")?, - resources, - query_count: integer(value, "query_count")?, - attributed_seconds: decimal(value, "attributed_seconds")?, - bytes_scanned: integer(value, "bytes_scanned")?, - estimated_compute_credits: decimal(value, "estimated_compute_credits")?, - estimated_usd: decimal(value, "estimated_usd")?, - expected_statement_count: integer(value, "expected_statement_count")?, - observed_statement_count: integer(value, "observed_statement_count")?, - missing_statement_count: integer(value, "missing_statement_count")?, - skipped_statement_count: integer(value, "skipped_statement_count")?, - source: text(value, "source")?, - method: text(value, "method")?, - limitations, - message: optional_text(value, "message")?, - }) -} - -/// A `CostRunRecord` as a plain run. -pub(crate) fn cost_run(record: &Bound<'_, PyAny>) -> PyResult { - Ok(CostRun { - run_id: text(record, "run_id")?, - adapter_name: text(record, "adapter_name")?, - target_name: optional_text(record, "target_name")?, - build_status: text(record, "build_status")?, - started_at: timestamp(record, "started_at")?, - completed_at: timestamp(record, "completed_at")?, - cost: summary(&native_field(record, "cost")?)?, - version: integer(record, "version")?, - had_executable_work: native_field(record, "had_executable_work")?.extract()?, - }) -} - -/// Every record of an iterable as a plain run. -pub(crate) fn cost_runs(records: &Bound<'_, PyAny>) -> PyResult> { - let mut runs: Vec = Vec::new(); - for record in records.try_iter()? { - runs.push(cost_run(&record?)?); - } - Ok(runs) -} - -/// Python constructors of the record fields. -struct Builders<'py> { - decimal: Bound<'py, PyAny>, - timestamp: Bound<'py, PyAny>, -} - -impl<'py> Builders<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - decimal: imported(py, "decimal", "Decimal")?, - timestamp: imported(py, "datetime", "datetime")?.getattr("fromisoformat")?, - }) - } - - fn decimal(&self, text: &str) -> PyResult> { - self.decimal.call1((text,)) - } -} - -fn resource_object<'py>( - builders: &Builders<'py>, - resource: &CostResource, -) -> PyResult> { - let py: Python<'py> = builders.decimal.py(); - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("resource_type", &resource.resource_type)?; - fields.set_item("resource_name", &resource.resource_name)?; - fields.set_item("warehouse_name", &resource.warehouse_name)?; - fields.set_item("warehouse_size", &resource.warehouse_size)?; - fields.set_item("warehouse_type", &resource.warehouse_type)?; - fields.set_item("cluster_number", resource.cluster_number)?; - fields.set_item("query_count", resource.query_count)?; - fields.set_item( - "attributed_seconds", - builders.decimal(&resource.attributed_seconds)?, - )?; - fields.set_item("bytes_scanned", resource.bytes_scanned)?; - fields.set_item( - "estimated_compute_credits", - builders.decimal(&resource.estimated_compute_credits)?, - )?; - fields.set_item("estimated_usd", builders.decimal(&resource.estimated_usd)?)?; - new_record(py, RecordId::ResourceCost, (), Some(&fields)) -} - -fn summary_object<'py>( - builders: &Builders<'py>, - cost: &CostSummary, -) -> PyResult> { - let py: Python<'py> = builders.decimal.py(); - let mut resources: Vec> = Vec::with_capacity(cost.resources.len()); - for resource in &cost.resources { - resources.push(resource_object(builders, resource)?); - } - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item( - "status", - host_entry(py, ExecutorHost::CostStatus)?.call1((cost.status.as_str(),))?, - )?; - fields.set_item("usd_per_credit", builders.decimal(&cost.usd_per_credit)?)?; - fields.set_item("rate_source", &cost.rate_source)?; - fields.set_item("resources", PyTuple::new(py, resources)?)?; - fields.set_item("query_count", cost.query_count)?; - fields.set_item( - "attributed_seconds", - builders.decimal(&cost.attributed_seconds)?, - )?; - fields.set_item("bytes_scanned", cost.bytes_scanned)?; - fields.set_item( - "estimated_compute_credits", - builders.decimal(&cost.estimated_compute_credits)?, - )?; - fields.set_item("estimated_usd", builders.decimal(&cost.estimated_usd)?)?; - fields.set_item("expected_statement_count", cost.expected_statement_count)?; - fields.set_item("observed_statement_count", cost.observed_statement_count)?; - fields.set_item("missing_statement_count", cost.missing_statement_count)?; - fields.set_item("skipped_statement_count", cost.skipped_statement_count)?; - fields.set_item("source", &cost.source)?; - fields.set_item("method", &cost.method)?; - fields.set_item("limitations", PyTuple::new(py, &cost.limitations)?)?; - fields.set_item("message", &cost.message)?; - host_entry(py, ExecutorHost::RunCostSummary)?.call((), Some(&fields)) -} - -/// A plain run as the `CostRunRecord` it was read from. -pub(crate) fn record_object<'py>(py: Python<'py>, run: &CostRun) -> PyResult> { - let builders: Builders<'py> = Builders::new(py)?; - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("version", run.version)?; - fields.set_item("run_id", &run.run_id)?; - fields.set_item("adapter_name", &run.adapter_name)?; - fields.set_item("target_name", &run.target_name)?; - fields.set_item("build_status", &run.build_status)?; - fields.set_item( - "started_at", - builders.timestamp.call1((run.started_at.as_str(),))?, - )?; - fields.set_item( - "completed_at", - builders.timestamp.call1((run.completed_at.as_str(),))?, - )?; - fields.set_item("cost", summary_object(&builders, &run.cost)?)?; - fields.set_item("had_executable_work", run.had_executable_work)?; - new_record(py, RecordId::CostRunRecord, (), Some(&fields)) -} - -/// Records of several plain runs, as a tuple. -pub(crate) fn record_tuple<'py>( - py: Python<'py>, - runs: &[CostRun], -) -> PyResult> { - let mut records: Vec> = Vec::with_capacity(runs.len()); - for run in runs { - records.push(record_object(py, run)?); - } - PyTuple::new(py, records) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs index ecc1512232..a45f150a11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs @@ -9,7 +9,7 @@ use sqlbuild_cli::cost_store::main::read_cost_run::read_cost_run; use sqlbuild_cli::cost_store::main::write_cost_run::write_cost_run; use sqlbuild_core::cost_report::models::CostRun; -use crate::bindings::_helpers::cost_report::records::artifact_error; +use sqlbuild_python_front::resource_costs::main::artifact_error::artifact_error; fn failure(py: Python<'_>, error: CostStoreError) -> PyErr { match error { diff --git a/fensu.toml b/fensu.toml index 9e9faafec2..4c7f00eae3 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2721,7 +2721,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs=9", @@ -3036,7 +3035,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=2", From b0794915af31b1fcaa3445051548865c5bb5da6f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:30:46 +0000 Subject: [PATCH 3/3] fix(cost): finish runs first and stop re-reading costs that cannot change Run-end cost collection kept builds and server runs open for 30-50 s and re-read the same runs on every build without changing them. - A finished query still without a warehouse size two minutes after it ended ran without warehouse compute (for example a result Snowflake served from metadata); it no longer marks the run partial. A newer one is still missing metadata, which a later build's re-read settles. - A partial run is re-read only while a statement was unseen, still running or awaiting metadata; a run that saw every statement with only permanent limitations is final. Runs are re-read only once they ended two minutes ago. - An older run's re-read reads only its own window instead of every query since it started, and re-reads share an 8 s budget per build. - The run-end sweep does not retry for missing metadata alone, and a command-line build's sweep has a 6 s budget. - A command-line build prints its summary, then 'Collecting warehouse costs...'. A worker build records its cost pending and collects it after the run publishes its outcome, so the run finishes and releases its claims first. --- .../src/cost_command/constants.rs | 9 ++ .../src/native_host/main/elapsed_seconds.rs | 8 ++ .../src/native_host/main/mod.rs | 3 + .../main/reports_cost_capability.rs | 15 +++ .../src/native_host/main/seconds_since.rs | 14 +++ .../_helpers/resource_costs_state.rs | 16 +-- .../src/resource_costs/models.rs | 6 +- .../src/cost_telemetry/_helpers/snowflake.rs | 48 ++++++--- .../src/native_host/models.rs | 27 +++++ .../src/project_snapshot/models.rs | 14 +++ .../_helpers/build_command/completion.rs | 101 ++++++++++++++---- .../src/bindings/_helpers/cli/entry.rs | 2 + .../bindings/_helpers/cost_report/deferred.rs | 25 +++++ .../_helpers/cost_report/degradation.rs | 18 +++- .../src/bindings/_helpers/cost_report/mod.rs | 1 + .../_helpers/cost_report/publication.rs | 15 ++- .../_helpers/location_command/worker.rs | 5 +- .../src/resource_costs/constants.rs | 4 + .../resource_costs/main/refresh_candidates.rs | 20 +++- .../src/resource_costs/models.rs | 2 + .../src/resource_costs/tests/helpers.rs | 9 ++ .../src/resource_costs/tests/test_refresh.rs | 30 +++++- tests/unit/src/sqlbuild/cost/_test_types.py | 8 ++ .../unit/src/sqlbuild/cost/test_snowflake.py | 87 +++++++++++++-- 24 files changed, 427 insertions(+), 60 deletions(-) create mode 100644 crates/sqlbuild-python-front/src/native_host/main/elapsed_seconds.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/reports_cost_capability.rs create mode 100644 crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_report/deferred.rs diff --git a/crates/sqlbuild-core/src/cost_command/constants.rs b/crates/sqlbuild-core/src/cost_command/constants.rs index 83f1bbc19b..f9b76b7636 100644 --- a/crates/sqlbuild-core/src/cost_command/constants.rs +++ b/crates/sqlbuild-core/src/cost_command/constants.rs @@ -27,3 +27,12 @@ pub const MISSING_METRIC_STATUSES: [&str; 4] = /// Statuses a delayed refresh may still improve. pub const REFRESHABLE_STATUSES: [&str; 2] = ["pending", "partial"]; pub const LEDGER_PERSISTENCE_FAILURE_PREFIX: &str = "Statement-ledger persistence failed ("; +/// The limitation of a run whose statements query history still showed running, which a re-read can complete. +pub const INCOMPLETE_HISTORY_LIMITATION: &str = + "One or more run statements were still incomplete in query history."; +/// The limitation of a run whose recent statements had no warehouse metadata yet, which a re-read can complete. +pub const MISSING_METADATA_LIMITATION: &str = + "One or more run statements had missing Snowflake warehouse metadata."; +/// The limitations a re-read of a run that saw every statement can still lift. +pub const CHANGEABLE_LIMITATIONS: [&str; 2] = + [INCOMPLETE_HISTORY_LIMITATION, MISSING_METADATA_LIMITATION]; diff --git a/crates/sqlbuild-python-front/src/native_host/main/elapsed_seconds.rs b/crates/sqlbuild-python-front/src/native_host/main/elapsed_seconds.rs new file mode 100644 index 0000000000..c8951e68ee --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/elapsed_seconds.rs @@ -0,0 +1,8 @@ +//! Elapsed time between two Python datetimes. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// Seconds from `from` to `to`, negative when `to` is earlier. +pub fn elapsed_seconds(from: &Bound<'_, PyAny>, to: &Bound<'_, PyAny>) -> PyResult { + to.sub(from)?.call_method0("total_seconds")?.extract() +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/mod.rs b/crates/sqlbuild-python-front/src/native_host/main/mod.rs index 4cb7f09303..393249f52a 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/mod.rs @@ -1,5 +1,6 @@ pub mod append_observations; pub mod append_replaced_consumption; +pub mod elapsed_seconds; pub mod execute; pub mod fixed_state_table_sql; pub mod integer_literal; @@ -9,4 +10,6 @@ pub mod recorder_callbacks; pub mod register_audit_results_state; pub mod register_consumption_state; pub mod register_observations_state; +pub mod reports_cost_capability; +pub mod seconds_since; pub mod state_table_name; diff --git a/crates/sqlbuild-python-front/src/native_host/main/reports_cost_capability.rs b/crates/sqlbuild-python-front/src/native_host/main/reports_cost_capability.rs new file mode 100644 index 0000000000..4b42260f7c --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/reports_cost_capability.rs @@ -0,0 +1,15 @@ +//! Whether a cost-aware adapter reports a cost capability. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// Whether `adapter` reports a cost capability; a failing report still counts as one. +pub fn reports_cost_capability(adapter: &Bound<'_, PyAny>) -> PyResult { + let none: Bound<'_, PyAny> = + host_entry(adapter.py(), ExecutorHost::CostCapability)?.getattr("NONE")?; + Ok(match adapter.call_method0("cost_capability") { + Ok(capability) => !capability.eq(none)?, + Err(_) => true, + }) +} diff --git a/crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs b/crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs new file mode 100644 index 0000000000..f4bce84a3e --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs @@ -0,0 +1,14 @@ +//! Time since a Python datetime, by the UTC clock now. + +use crate::cursor::constants::DATETIME_MODULE; +use crate::native_host::main::elapsed_seconds::elapsed_seconds; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// Seconds since `value`, by the UTC clock now. +pub fn seconds_since(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let now: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")? + .call_method1("now", (imported(py, DATETIME_MODULE, "UTC")?,))?; + elapsed_seconds(value, &now) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs index 347a1823b2..50a1ae3831 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs @@ -31,8 +31,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; const CONFIGURED_RATE_SOURCE: &str = "configured"; /// The adapter method that marks the resource costs table as supported. pub(crate) const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql"; -/// Collection budget of each re-read of an older run, which a build waits for. -pub(crate) const REFRESH_BUDGET_SECONDS: f64 = 20.0; /// Create-if-missing DDL for the resource costs table, with adapter-rendered column types. #[pyfunction] @@ -210,9 +208,14 @@ pub(crate) fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { let limitations: Vec = serde_json::from_str(&text(7)?) .map_err(|error| PyValueError::new_err(format!("unreadable limitations: {error}")))?; let run_started_at: String = utc_text(&cell(9)?)?; - let started_epoch: i64 = DateTime::parse_from_rfc3339(&run_started_at) - .map_err(|error| PyValueError::new_err(error.to_string()))? - .timestamp(); + let epoch = |text: &str| -> PyResult { + Ok(DateTime::parse_from_rfc3339(text) + .map_err(|error| PyValueError::new_err(error.to_string()))? + .timestamp()) + }; + let started_epoch: i64 = epoch(&run_started_at)?; + let run_completed_at: String = utc_text(&cell(10)?)?; + let completed_epoch: i64 = epoch(&run_completed_at)?; Ok(RunCostAttempt { run_id: text(0)?, orchestration_run_id: optional_text(&cell(1)?)?, @@ -226,7 +229,8 @@ pub(crate) fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { }, target_name: optional_text(&cell(8)?)?, run_started_at, - run_completed_at: utc_text(&cell(10)?)?, + run_completed_at, started_epoch, + completed_epoch, }) } diff --git a/crates/sqlbuild-python-front/src/resource_costs/models.rs b/crates/sqlbuild-python-front/src/resource_costs/models.rs index 6fb02bc785..5bed834d05 100644 --- a/crates/sqlbuild-python-front/src/resource_costs/models.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/models.rs @@ -4,7 +4,6 @@ use crate::native_host::constants::ROWS_PER_STATEMENT; use crate::native_host::main::execute::execute; use crate::native_host::models::Location; use crate::resource_costs::_helpers::resource_costs_state::CREATE_TABLE_METHOD; -use crate::resource_costs::_helpers::resource_costs_state::REFRESH_BUDGET_SECONDS; use crate::resource_costs::_helpers::resource_costs_state::observation_cells; use crate::resource_costs::_helpers::resource_costs_state::run_attempt; use crate::resource_costs::_helpers::resource_costs_state::table; @@ -130,11 +129,12 @@ impl<'py> ResourceCostStore<'py> { .collect() } - /// Re-collect one older run through the adapter, priced at `rate`. + /// Re-collect one older run through the adapter, priced at `rate`, within `budget_seconds`. pub fn recollect( &self, run: &RunCostAttempt, (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + budget_seconds: f64, ) -> PyResult> { let py: Python<'py> = self.adapter.py(); let parse = imported(py, "datetime", "datetime")?.getattr("fromisoformat")?; @@ -165,7 +165,7 @@ impl<'py> ResourceCostStore<'py> { ("rate_source", rate_source.clone()), ( "budget_seconds", - PyFloat::new(py, REFRESH_BUDGET_SECONDS).into_any(), + PyFloat::new(py, budget_seconds).into_any(), ), ], ) diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs index 78824e1588..e8d0910251 100644 --- a/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs +++ b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs @@ -12,6 +12,9 @@ use sqlbuild_analysis::snowflake_cost::models::QueryTag; use sqlbuild_core::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; use sqlbuild_core::cost_allocation::main::query_history_sql::query_history_sql; use sqlbuild_core::cost_allocation::models::{QueryHistoryWindow, RunAllocation}; +use sqlbuild_core::cost_command::constants::{ + INCOMPLETE_HISTORY_LIMITATION, MISSING_METADATA_LIMITATION, +}; use sqlbuild_host::entries::models::ExecutorHost; use crate::cost_telemetry::_helpers::allocation::SummaryState; @@ -29,6 +32,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::native_host::main::elapsed_seconds::elapsed_seconds; +use sqlbuild_python_front::native_host::main::seconds_since::seconds_since; /// Default budget of one collection, which the at-exit collection keeps short. const DEADLINE_SECONDS: f64 = 30.0; @@ -40,6 +45,8 @@ const STATEMENT_TIMEOUT_CODE: &str = "000630"; const RUNNING_STATUS: &str = "RUNNING"; const HOOK_RESOURCE_TYPE: &str = "hook"; const ROW_WIDTH: usize = 14; +/// Seconds after which a finished query still without a warehouse size, such as DDL, used none. +const METADATA_SETTLE_SECONDS: f64 = 120.0; const NO_WAREHOUSE_COMPUTE: &str = "no_warehouse_compute"; const MISSING_WAREHOUSE_METADATA: &str = "missing_warehouse_metadata"; const INCOMPLETE_HISTORY: &str = "incomplete_history"; @@ -311,7 +318,13 @@ fn row_observation<'py>( return Ok(Err(INCOMPLETE_HISTORY)); } if milliseconds.eq(0)? || warehouse_name.is_none() || warehouse_size.is_none() { - return Ok(Err(MISSING_WAREHOUSE_METADATA)); + let settled: bool = + !running && elapsed_seconds(&end_time, &observed_at)? >= METADATA_SETTLE_SECONDS; + return Ok(Err(if settled { + NO_WAREHOUSE_COMPUTE + } else { + MISSING_WAREHOUSE_METADATA + })); } let tag: Option = if query_tag.is_instance_of::() { sqlbuild_query_tag(&query_tag.extract::()?) @@ -392,7 +405,7 @@ fn open_rows<'py>( )) } -/// Observations of the run window: one open read, halved windows when saturated or timed out. +/// Observations of the run window: an open read while it is open, else only the window itself. fn query_observations<'py>( history: &History<'py>, (started_at, completed_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -402,7 +415,11 @@ fn query_observations<'py>( let skew: Bound<'py, PyAny> = timedelta(py, ("minutes", 10))?; let window_start: Bound<'py, PyAny> = started_at.sub(&skew)?; let window_end: Bound<'py, PyAny> = completed_at.add(&skew)?; - let (rows, saturated) = open_rows(history, (&window_start, &window_end))?; + let (rows, saturated) = if seconds_since(&window_end)?.is_sign_positive() { + bounded_rows(history, (&window_start, &window_end))? + } else { + open_rows(history, (&window_start, &window_end))? + }; let observations: Bound<'py, PyList> = PyList::empty(py); let mut observed: HashSet = HashSet::new(); let mut skipped: Vec<(String, &'static str)> = Vec::new(); @@ -531,12 +548,10 @@ fn with_observation_gaps( let incomplete: bool = relevant.contains(&INCOMPLETE_HISTORY); let mut added: Vec = Vec::new(); if missing_metadata { - added.push( - "One or more run statements had missing Snowflake warehouse metadata.".to_owned(), - ); + added.push(MISSING_METADATA_LIMITATION.to_owned()); } if incomplete { - added.push("One or more run statements were still incomplete in query history.".to_owned()); + added.push(INCOMPLETE_HISTORY_LIMITATION.to_owned()); } summary.state.limitations = with_limitations(&summary.state.limitations, added); let query_count: usize = summary @@ -556,6 +571,15 @@ fn with_observation_gaps( summary } +/// Whether every expected statement is visible and finished, lacking at most settling metadata. +fn awaits_only_metadata(expected: &HashSet, observed: &Observed<'_>) -> bool { + expected.is_subset(&observed.observed) + && !observed + .skipped + .iter() + .any(|(id, reason)| expected.contains(id) && *reason == INCOMPLETE_HISTORY) +} + fn incomplete_observation(expected: &HashSet, observed: &Observed<'_>) -> bool { observed.skipped.iter().any(|(id, reason)| { expected.contains(id) @@ -690,12 +714,12 @@ fn collect<'py>( return Ok(with_ledger_failure(complete, failure.as_deref())); } summary = Some(current); - if attempt + 1 < collection.attempts { - if history.deadline - monotonic(py)? <= collection.retry_delay_seconds { - break; - } - imported(py, "time", "sleep")?.call1((collection.retry_delay_seconds,))?; + let last: bool = + attempt + 1 >= collection.attempts || awaits_only_metadata(&expected_ids, &observed); + if last || history.deadline - monotonic(py)? <= collection.retry_delay_seconds { + break; } + imported(py, "time", "sleep")?.call1((collection.retry_delay_seconds,))?; } let mut summary: Summary = summary.unwrap_or_else(|| partial("Snowflake query history could not be collected.")); diff --git a/crates/sqlbuild-python-project/src/native_host/models.rs b/crates/sqlbuild-python-project/src/native_host/models.rs index 415afdd71a..70a2b87d14 100644 --- a/crates/sqlbuild-python-project/src/native_host/models.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -97,3 +97,30 @@ pub struct CalendarRequest { pub timezone: String, pub trigger: TriggerDefinition, } + +/// A worker build's cost finalization, held until the run has published its outcome. +#[derive(Debug, Default)] +pub struct DeferredCost(std::sync::Mutex>>); + +impl DeferredCost { + const POISONED: &str = "deferred cost lock is poisoned"; + + /// Keep `finalization`, replacing any earlier one. + pub fn defer(&self, finalization: &Bound<'_, PyAny>) -> PyResult<()> { + *self + .0 + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err(Self::POISONED))? = + Some(finalization.clone().unbind()); + Ok(()) + } + + /// The deferred finalization, once. + pub fn take(&self) -> PyResult>> { + Ok(self + .0 + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err(Self::POISONED))? + .take()) + } +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/models.rs b/crates/sqlbuild-python-project/src/project_snapshot/models.rs index 7ec951f026..385f0b8903 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/models.rs @@ -5,6 +5,7 @@ use crate::compiled_project::constants::REPORT_ATTEMPTS; use crate::compiled_project::constants::SKIP_ACTION; use crate::compiled_project::constants::SQL_TEST_KEY_PREFIX; use crate::compiled_project::constants::STORE_BUSY_STATUS; +use crate::native_host::models::DeferredCost; use crate::project_snapshot::_helpers::snapshot::snapshot_execution::Completion; use crate::project_snapshot::_helpers::snapshot::snapshot_execution::is_function; use crate::project_snapshot::_helpers::snapshot::snapshot_execution::poisoned; @@ -142,6 +143,8 @@ pub struct WorkerBuild { refusal: Mutex>, /// Error-severity audit failures the build let through, reported before the run finishes. ignored_audit_failures: AtomicU32, + /// The build's cost finalization, collected once the run has reported its outcome. + deferred_cost: DeferredCost, } impl WorkerBuild { @@ -164,6 +167,7 @@ impl WorkerBuild { returned: AtomicBool::new(false), ignored_audit_failures: AtomicU32::new(0), refusal: Mutex::default(), + deferred_cost: DeferredCost::default(), }); for key in state.execution.skipped_nodes.keys() { state.complete(worker.py(), key, (NodeOutcome::Skipped, None, (None, None)))?; @@ -180,6 +184,16 @@ impl WorkerBuild { self.execution.claim.request.run_id } + /// Keep the build's cost finalization to collect after the run's outcome is published. + pub fn defer_cost(&self, finalization: &Bound<'_, PyAny>) -> PyResult<()> { + self.deferred_cost.defer(finalization) + } + + /// The cost finalization the build deferred, once. + pub fn take_deferred_cost(&self) -> PyResult>> { + self.deferred_cost.take() + } + /// The build let `count` error-severity audit failures through. pub fn ignored_audit_failures(&self, count: i64) { self.ignored_audit_failures.store( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 60837a78cc..c7e486bdbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -17,7 +17,9 @@ use crate::bindings::_helpers::build_command::execution::Preparation; use crate::bindings::_helpers::build_command::execution::write_run_context; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::check::check_results::check_result_text; +use crate::bindings::_helpers::cost_report::degradation::RUN_END_SWEEP_SECONDS; use crate::bindings::_helpers::cost_report::degradation::finalize_build_cost as finalize_cost; +use crate::bindings::_helpers::cost_report::degradation::finalize_build_cost_within as finalize_cost_within; use crate::bindings::_helpers::cost_report::degradation::render_build_cost; use crate::bindings::_helpers::cost_report::functions::cost_output_payload; use sqlbuild_core::build_progress::models::PythonNodeText; @@ -37,6 +39,8 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::plan_has_executable_work::plan_has_executable_work; use sqlbuild_python_compile::compile_outputs::main::write_python_check_runtime_target::write_python_check_runtime_target; use sqlbuild_python_compile::compile_outputs::main::write_runtime_target::write_runtime_target; +use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; +use sqlbuild_python_front::native_host::main::reports_cost_capability::reports_cost_capability; use sqlbuild_python_output::build_progress::main::build_result_text::build_result_text; use sqlbuild_python_output::build_progress::main::format_build_footer_with::format_build_footer_with; use sqlbuild_python_output::result_output::main::results::format_build_execution_json::format_build_execution_json; @@ -48,6 +52,8 @@ const CHECK_KIND: &str = "check"; const CHECK_WARNED: &str = "check warned"; const PENDING_MESSAGE: &str = "Build cost collection has not completed."; const INTERRUPTED_MESSAGE: &str = "Build was interrupted before cost collection completed."; +const DEFERRED_MESSAGE: &str = "Warehouse costs are collected after the run finishes."; +const COLLECTING_LINE: &str = "Collecting warehouse costs...\n"; /// How one cost finalization differs from a completed collection. pub(crate) struct CostRequest<'py> { @@ -62,6 +68,15 @@ pub(crate) fn finalize_build_cost<'py>( context: &BuildContext<'py>, pipeline: &Bound<'py, PyAny>, cost: CostRequest<'py>, +) -> PyResult> { + finalize_cost(&cost_finalization(context, pipeline, cost)?) +} + +/// The `BuildCostFinalization` of one cost request. +fn cost_finalization<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + cost: CostRequest<'py>, ) -> PyResult> { let py: Python<'py> = context.py(); let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; @@ -103,9 +118,56 @@ pub(crate) fn finalize_build_cost<'py>( )); } fields.extend(cost.overrides); - let finalization: Bound<'py, PyAny> = - new_record_with(py, RecordId::BuildCostFinalization, &fields)?; - finalize_cost(&finalization) + new_record_with(py, RecordId::BuildCostFinalization, &fields) +} + +/// A finished build's cost: a worker's waits until its run has finished; the CLI sweeps briefly. +fn finished_build_cost<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + (status, started_at, completed_at): (&'static str, &Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = context.py(); + let request = |overrides: Vec<(&'static str, Bound<'py, PyAny>)>| CostRequest { + status, + started_at: started_at.clone(), + completed_at: completed_at.clone(), + overrides, + }; + let adapter: Bound<'py, PyAny> = context.get("adapter")?; + if !cost_aware(&adapter)? || !reports_cost_capability(&adapter)? { + return finalize_build_cost( + context, + pipeline, + request(vec![("render", boolean(py, false))]), + ); + } + if let Some(worker) = &context.worker { + worker.defer_cost(&cost_finalization( + context, + pipeline, + request(vec![("render", boolean(py, false))]), + )?)?; + return finalize_build_cost( + context, + pipeline, + request(vec![ + ("collect", boolean(py, false)), + ("render", boolean(py, false)), + ("cost_status", cost_status(py, "PENDING")?), + ("cost_message", text(py, DEFERRED_MESSAGE)), + ]), + ); + } + emit(&context.stream, COLLECTING_LINE)?; + finalize_cost_within( + &cost_finalization( + context, + pipeline, + request(vec![("render", boolean(py, false))]), + )?, + Some(RUN_END_SWEEP_SECONDS), + ) } fn cost_status<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -384,23 +446,6 @@ pub(crate) fn complete_direct_build<'py>( joined(&context.get("effective_project_dir")?, &["target"])?; write_runtime_target(&target_dir, &plan, result)?; write_python_check_runtime_target(&target_dir, check_results)?; - let cost_started: Instant = Instant::now(); - let record: PyResult> = finalize_build_cost( - context, - pipeline, - CostRequest { - status: if exit_code == 0 { - SUCCESS_STATUS - } else { - "failed" - }, - started_at: started_at.clone(), - completed_at, - overrides: vec![("render", boolean(py, false))], - }, - ); - let cost_seconds: f64 = finish_cost_timing(context, cost_started); - let record: Bound<'py, PyAny> = record?; let checks: Vec = check_results .try_iter()? .map(|check| Ok(check_node_text(&check_result_text(&check?)?))) @@ -415,6 +460,22 @@ pub(crate) fn complete_direct_build<'py>( checks, )?; emit(&context.stream, &format!("\n{footer}\n"))?; + let cost_started: Instant = Instant::now(); + let record: PyResult> = finished_build_cost( + context, + pipeline, + ( + if exit_code == 0 { + SUCCESS_STATUS + } else { + "failed" + }, + started_at, + completed_at, + ), + ); + let cost_seconds: f64 = finish_cost_timing(context, cost_started); + let record: Bound<'py, PyAny> = record?; let payload: Bound<'py, PyAny> = format_build_execution_json_with( py, result, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 194c02db2e..8636c86891 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -35,6 +35,7 @@ use crate::bindings::_helpers::cli::termination::{ use sqlbuild_python_compile::compile_command::constants::COMPILE_COMMAND_NAME; use sqlbuild_python_compile::compile_command::main::invocation::format_compile_error_json::format_compile_error_json; use crate::bindings::_helpers::cost_report::attribution::take_cost_status; +use crate::bindings::_helpers::cost_report::deferred::collect_deferred_cost_after_finish; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use sqlbuild_python_base::native_host::models::InterruptHandler; use sqlbuild_python_base::native_host::main::interrupted::interrupted; @@ -933,6 +934,7 @@ fn execute_parsed_invocation<'py>( py, &format!("Worker execution completed: {}\n", outcome.as_str()), )?; + collect_deferred_cost_after_finish(py, &worker); if !matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { dispatched?; finished?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/deferred.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/deferred.rs new file mode 100644 index 0000000000..46627a3ea9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/deferred.rs @@ -0,0 +1,25 @@ +//! A worker build's warehouse costs, collected after the run reports its outcome so the run is not held open. + +use pyo3::prelude::{PyResult, Python}; + +use crate::bindings::_helpers::cost_report::degradation::finalize_build_cost_within; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; + +/// Collect a deferred build's costs after its run finished; a failure only warns, as the outcome is published. +pub(crate) fn collect_deferred_cost_after_finish(py: Python<'_>, worker: &WorkerBuild) { + if let Err(error) = collect_deferred_cost(py, worker) { + let _ = write_stderr( + py, + &format!("warning: warehouse cost collection failed: {error}\n"), + ); + } +} + +fn collect_deferred_cost(py: Python<'_>, worker: &WorkerBuild) -> PyResult<()> { + let Some(finalization) = worker.take_deferred_cost()? else { + return Ok(()); + }; + write_stderr(py, "Collecting warehouse costs...\n")?; + finalize_build_cost_within(finalization.bind(py), None).map(|_| ()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 6949d5c776..a89ee28dfd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -30,6 +30,8 @@ const CONFIGURED_RATE_SOURCE: &str = "configured"; const STATEMENT_LEDGER_PATH: [&str; 4] = ["target", "executions", "", "statements.jsonl"]; const BUILD_BREAKDOWN_LIMIT: usize = 5; const FALLBACK_TERMINAL_COLUMNS: usize = 100; +/// Query-history budget of a command-line build's run-end sweep; what it misses, a later build reads. +pub(crate) const RUN_END_SWEEP_SECONDS: f64 = 6.0; /// Log the failure with its traceback, as `_LOGGER.exception` inside the handler did. fn log_failure(py: Python<'_>, message: &str, error: &PyErr) -> PyResult<()> { @@ -271,7 +273,7 @@ fn collection_failure<'py>( fn collected_summary<'py>( finalization: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, - (config, rate_source): (&Bound<'py, PyAny>, &str), + ((config, rate_source), budget): ((&Bound<'py, PyAny>, &str), Option), ) -> PyResult> { let py: Python<'py> = finalization.py(); let run_id: Bound<'py, PyAny> = native_field(finalization, "run_id")?; @@ -300,6 +302,10 @@ fn collected_summary<'py>( ("statement_ledger_path", ledger), ("usd_per_credit", native_field(config, "usd_per_credit")?), ("rate_source", PyString::new(py, rate_source).into_any()), + ( + "budget_seconds", + pyo3::IntoPyObjectExt::into_bound_py_any(budget, py)?, + ), ], ) } @@ -308,6 +314,14 @@ fn collected_summary<'py>( #[pyfunction] pub(crate) fn finalize_build_cost<'py>( finalization: &Bound<'py, PyAny>, +) -> PyResult> { + finalize_build_cost_within(finalization, None) +} + +/// [`finalize_build_cost`] with the collection's query-history budget, or the adapter's default. +pub(crate) fn finalize_build_cost_within<'py>( + finalization: &Bound<'py, PyAny>, + budget: Option, ) -> PyResult> { let py: Python<'py> = finalization.py(); let adapter: Bound<'py, PyAny> = native_field(finalization, "adapter")?; @@ -331,7 +345,7 @@ pub(crate) fn finalize_build_cost<'py>( return Ok(py.None().into_bound(py)); } let cost: Bound<'py, PyAny> = if native_field(finalization, "collect")?.is_truthy()? { - match collected_summary(finalization, &adapter, rate) { + match collected_summary(finalization, &adapter, (rate, budget)) { Ok(summary) => summary, Err(error) => return collection_failure(finalization, rate, "collection", &error), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index 81fa2bd091..d5d28ef528 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod attribution; +pub(crate) mod deferred; pub(crate) mod degradation; pub(crate) mod exception_types; pub(crate) mod functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs index 6126b5b574..74c03b1b62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs @@ -1,6 +1,6 @@ //! Record a build's run cost in the target's state schema, then re-read older incomplete runs. -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -18,6 +18,12 @@ use sqlbuild_python_front::resource_costs::models::ResourceCostStore; /// The attempt number of a build's own collection; each accepted re-read appends the next. const FIRST_ATTEMPT: i64 = 1; +/// Query-history time a build spends re-reading older runs, across all of them. +const REFRESH_TOTAL_SECONDS: f64 = 8.0; +/// Query-history time one older run's re-read may take. +const REFRESH_RUN_SECONDS: f64 = 5.0; +/// Remaining time below which no further older run is re-read. +const REFRESH_MIN_SECONDS: f64 = 1.0; fn coverage(summary: &CostSummary) -> CostCoverage { CostCoverage { @@ -47,9 +53,14 @@ fn refresh_older( .into_iter() .filter(|latest| latest.run_id != run_id) .collect(); + let started: Instant = Instant::now(); for older in refresh_candidates(&latest, unix_now()?) { + let remaining: f64 = REFRESH_TOTAL_SECONDS - started.elapsed().as_secs_f64(); + if remaining < REFRESH_MIN_SECONDS { + break; + } let Ok(refreshed) = store - .recollect(&older, rate) + .recollect(&older, rate, remaining.min(REFRESH_RUN_SECONDS)) .and_then(|collected| summary(&collected)) else { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 7a88a487a1..2bf84302ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -29,6 +29,7 @@ use crate::bindings::_helpers::cli::dispatch::warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::WarehouseInputs; use crate::bindings::_helpers::command_runtime::warehouses::configured_warehouses; use crate::bindings::_helpers::compile_driver::rules_at_load::RulesAtLoadScope; +use crate::bindings::_helpers::cost_report::deferred::collect_deferred_cost_after_finish; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; use crate::bindings::_helpers::location_command::constants::CHANGED_SETTING; @@ -396,7 +397,9 @@ pub(super) fn execute_worker<'py>( let outcome = if recall.is_some() { worker.outcome(py, code)? } else { - worker.finish(py, code)? + let outcome = worker.finish(py, code)?; + collect_deferred_cost_after_finish(py, &worker); + outcome }; result.map(|code| { if matches!(outcome, RunOutcome::Success | RunOutcome::Warn) { diff --git a/crates/sqlbuild-state/src/resource_costs/constants.rs b/crates/sqlbuild-state/src/resource_costs/constants.rs index 4ce0fe26be..957ca4838b 100644 --- a/crates/sqlbuild-state/src/resource_costs/constants.rs +++ b/crates/sqlbuild-state/src/resource_costs/constants.rs @@ -59,6 +59,8 @@ pub const LATEST_RUN_COST_COLUMNS: [&str; 11] = [ pub const RESOURCE_COST_TAG: &str = "resource_cost"; /// Statuses a later build re-reads while the warehouse is awake. pub const REFRESHABLE_COST_STATUSES: [&str; 3] = ["pending", "partial", "collection_failed"]; +/// Status re-read only while a statement was not visible yet or still running in query history. +pub const PARTIAL_STATUS: &str = "partial"; /// Status whose re-reads are bounded by `MAX_FAILED_COLLECTION_ATTEMPTS`. pub const COLLECTION_FAILED_STATUS: &str = "collection_failed"; /// Collections of a run, the first included, after which a failed collection is not retried. @@ -67,5 +69,7 @@ pub const MAX_FAILED_COLLECTION_ATTEMPTS: i64 = 3; pub const REFRESH_WINDOW_DAYS: i64 = 6; /// Most runs one build re-reads, newest first. pub const MAX_REFRESHED_RUNS: usize = 3; +/// Seconds after a run ends before a later build re-reads it, once its history has settled. +pub const REFRESH_SETTLE_SECONDS: i64 = 120; /// Seconds in one day, for the refresh window. pub const SECONDS_PER_DAY: i64 = 86_400; diff --git a/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs index 0396acc699..f4d7b18cf5 100644 --- a/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs +++ b/crates/sqlbuild-state/src/resource_costs/main/refresh_candidates.rs @@ -1,12 +1,24 @@ //! Which runs a build re-reads while the warehouse is awake. +use sqlbuild_core::cost_command::constants::CHANGEABLE_LIMITATIONS; +use sqlbuild_core::cost_command::models::CostCoverage; + use crate::resource_costs::constants::{ - COLLECTION_FAILED_STATUS, MAX_FAILED_COLLECTION_ATTEMPTS, MAX_REFRESHED_RUNS, - REFRESH_WINDOW_DAYS, REFRESHABLE_COST_STATUSES, SECONDS_PER_DAY, + COLLECTION_FAILED_STATUS, MAX_FAILED_COLLECTION_ATTEMPTS, MAX_REFRESHED_RUNS, PARTIAL_STATUS, + REFRESH_SETTLE_SECONDS, REFRESH_WINDOW_DAYS, REFRESHABLE_COST_STATUSES, SECONDS_PER_DAY, }; use crate::resource_costs::models::RunCostAttempt; -/// Newest refreshable runs in the history window, skipping exhausted failures; `now` is epoch seconds. +/// Whether a partial run can still change: a statement unseen, running or lacking settling metadata. +fn partial_can_change(coverage: &CostCoverage) -> bool { + coverage.observed_statement_count < coverage.expected_statement_count + || coverage + .limitations + .iter() + .any(|limitation| CHANGEABLE_LIMITATIONS.contains(&limitation.as_str())) +} + +/// Newest settled runs a re-read can still change, skipping exhausted failures; `now` is epoch seconds. #[must_use] pub fn refresh_candidates(latest: &[RunCostAttempt], now: i64) -> Vec { let oldest: i64 = now - REFRESH_WINDOW_DAYS * SECONDS_PER_DAY; @@ -14,6 +26,8 @@ pub fn refresh_candidates(latest: &[RunCostAttempt], now: i64) -> Vec= oldest) + .filter(|run| now - run.completed_epoch >= REFRESH_SETTLE_SECONDS) + .filter(|run| run.coverage.status != PARTIAL_STATUS || partial_can_change(&run.coverage)) .filter(|run| { run.coverage.status != COLLECTION_FAILED_STATUS || run.attempt < MAX_FAILED_COLLECTION_ATTEMPTS diff --git a/crates/sqlbuild-state/src/resource_costs/models.rs b/crates/sqlbuild-state/src/resource_costs/models.rs index f1d400938d..2cc0f317ec 100644 --- a/crates/sqlbuild-state/src/resource_costs/models.rs +++ b/crates/sqlbuild-state/src/resource_costs/models.rs @@ -16,6 +16,8 @@ pub struct RunCostAttempt { pub run_completed_at: String, /// Seconds since the epoch of the run's start. pub started_epoch: i64, + /// Seconds since the epoch of the run's end. + pub completed_epoch: i64, } /// What identifies one observation: the run, the collection attempt, the resource and its warehouse. diff --git a/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs b/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs index 49237b71ee..0b53533fe8 100644 --- a/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs +++ b/crates/sqlbuild-state/src/resource_costs/tests/helpers.rs @@ -24,9 +24,18 @@ pub(crate) fn run(run_id: &str, attempt: i64, status: &str, age: i64) -> RunCost run_started_at: String::new(), run_completed_at: String::new(), started_epoch: NOW - age, + completed_epoch: NOW - age + 60, } } +/// A partial run that saw every statement, with `limitations`. +pub(crate) fn fully_observed(run_id: &str, limitations: &[&str]) -> RunCostAttempt { + let mut attempt: RunCostAttempt = run(run_id, 1, "partial", 3_600); + attempt.coverage.observed_statement_count = attempt.coverage.expected_statement_count; + attempt.coverage.limitations = limitations.iter().map(|text| (*text).to_owned()).collect(); + attempt +} + pub(crate) fn observation_id(attempt: i64, cluster_number: Option) -> String { resource_cost_observation_id(&ResourceCostKey { run_id: "run", diff --git a/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs index fb9ef8b77c..19de16e4d3 100644 --- a/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs +++ b/crates/sqlbuild-state/src/resource_costs/tests/test_refresh.rs @@ -1,5 +1,9 @@ use crate::resource_costs::main::refresh_candidates::refresh_candidates; -use crate::resource_costs::tests::helpers::{NOW, observation_id, run}; +use sqlbuild_core::cost_command::constants::{ + INCOMPLETE_HISTORY_LIMITATION, MISSING_METADATA_LIMITATION, +}; + +use crate::resource_costs::tests::helpers::{NOW, fully_observed, observation_id, run}; use crate::resource_costs::tests::test_types::{ObservationIdTestCase, RefreshTestCase}; const HOUR: i64 = 3_600; @@ -18,6 +22,30 @@ fn given_latest_run_rows_when_choosing_refreshes_then_only_bounded_recent_incomp ], expected_runs: vec!["pending", "partial"], }, + RefreshTestCase { + description: "a partial run that saw every statement is final unless history can still fill in", + latest: vec![ + fully_observed( + "permanent", + &[ + "Concurrency sharing includes only queries visible to the executing Snowflake role.", + "Snowflake query-history result limit was reached.", + ], + ), + fully_observed("still_running", &[INCOMPLETE_HISTORY_LIMITATION]), + fully_observed("recent_metadata", &[MISSING_METADATA_LIMITATION]), + run("unseen_statement", 1, "partial", 2 * HOUR), + ], + expected_runs: vec!["recent_metadata", "still_running", "unseen_statement"], + }, + RefreshTestCase { + description: "a run that ended under two minutes ago has not settled yet", + latest: vec![ + run("just_ended", 1, "pending", 150), + run("settled", 1, "pending", 180), + ], + expected_runs: vec!["settled"], + }, RefreshTestCase { description: "a failed collection is retried until its third attempt, not after", latest: vec![ diff --git a/tests/unit/src/sqlbuild/cost/_test_types.py b/tests/unit/src/sqlbuild/cost/_test_types.py index 145f865885..8ddcec4784 100644 --- a/tests/unit/src/sqlbuild/cost/_test_types.py +++ b/tests/unit/src/sqlbuild/cost/_test_types.py @@ -46,6 +46,7 @@ class SnowflakeObservationTestCase: expected_limitation_fragment: str = "" query_type: str = "SELECT" query_text: str = "SELECT * FROM orders" + observed_after_seconds: int = 0 @dataclass(frozen=True) @@ -147,3 +148,10 @@ class SnowflakeCostBudgetTestCase: seconds_per_read: float expected_status: CostStatus expected_message_fragment: str = "" + + +@dataclass(frozen=True) +class HistoryWindowTestCase: + description: str + completed_minutes_ago: int + expected_bounded: bool diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake.py b/tests/unit/src/sqlbuild/cost/test_snowflake.py index fabafcbfca..b2fe35676b 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake.py @@ -21,6 +21,7 @@ from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import ( CollectSnowflakeCostTestCase, + HistoryWindowTestCase, ParseQueryTagTestCase, RenderQueryHistorySqlTestCase, SnowflakeClockSkewTestCase, @@ -227,7 +228,7 @@ def test_given_exact_create_hook_when_collecting_then_hook_query_text_is_not_req def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not_saturated( test_case: SnowflakeSaturationTestCase, ) -> None: - origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) + origin: datetime = datetime.now(UTC) - timedelta(minutes=1) row: tuple[Any, ...] = ( "q-open", "", @@ -564,6 +565,37 @@ def test_given_ledger_persistence_failure_when_collecting_then_summary_is_partia expected_query_count=0, expected_limitation_fragment="missing Snowflake warehouse metadata", ), + SnowflakeObservationTestCase( + description="a settled query without warehouse size used no warehouse", + warehouse_name="DEV_WH", + warehouse_size=None, + execution_ms=2, + execution_status="SUCCESS", + expected_status=CostStatus.COMPLETE, + expected_query_count=0, + observed_after_seconds=121, + ), + SnowflakeObservationTestCase( + description="a settled query with zero execution used no warehouse", + warehouse_name="DEV_WH", + warehouse_size="X-Small", + execution_ms=0, + execution_status="SUCCESS", + expected_status=CostStatus.COMPLETE, + expected_query_count=0, + observed_after_seconds=121, + ), + SnowflakeObservationTestCase( + description="a query read just before it settles still lacks metadata", + warehouse_name="DEV_WH", + warehouse_size=None, + execution_ms=1000, + execution_status="SUCCESS", + expected_status=CostStatus.PARTIAL, + expected_query_count=0, + expected_limitation_fragment="missing Snowflake warehouse metadata", + observed_after_seconds=119, + ), SnowflakeObservationTestCase( description="cancelled cost-bearing statement remains attributable", warehouse_name="DEV_WH", @@ -642,7 +674,7 @@ def test_given_special_history_row_when_collecting_then_coverage_is_explicit( test_case.execution_status, test_case.query_type, test_case.query_text, - origin + timedelta(seconds=1), + origin + timedelta(seconds=1 + test_case.observed_after_seconds), ), ) @@ -741,15 +773,16 @@ def test_given_metadata_only_create_when_warehouse_size_is_absent_then_cost_is_c "test_case", [ CollectSnowflakeCostTestCase( - description="delayed insert metadata is retried", - expected_status=CostStatus.COMPLETE, - expected_query_count=1, - expected_resource_name="countries", + description="delayed insert metadata is left for a later build's re-read", + expected_status=CostStatus.PARTIAL, + expected_query_count=0, + expected_resource_name="", + expected_limitation_fragment="missing Snowflake warehouse metadata", ) ], ids=lambda case: case.description, ) -def test_given_compute_query_with_delayed_metadata_when_collecting_then_history_is_retried( +def test_given_compute_query_with_delayed_metadata_when_collecting_then_it_is_not_retried_at_run_end( test_case: CollectSnowflakeCostTestCase, tmp_path: Path, ) -> None: @@ -808,10 +841,10 @@ def test_given_compute_query_with_delayed_metadata_when_collecting_then_history_ retry_delay_seconds=0, ) - assert connection.call_count == 2 + assert connection.call_count == 1 assert summary.status == test_case.expected_status assert summary.query_count == test_case.expected_query_count - assert summary.resources[0].resource_name == test_case.expected_resource_name + assert test_case.expected_limitation_fragment in " ".join(summary.limitations) @pytest.mark.parametrize( @@ -1168,7 +1201,7 @@ def test_given_saturated_open_history_when_bounded_fallback_completes_then_runni test_case: CollectSnowflakeCostTestCase, tmp_path: Path, ) -> None: - origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) + origin: datetime = datetime.now(UTC) - timedelta(minutes=1) ledger_path: Path = tmp_path / "statements.jsonl" ledger_path.write_text( json.dumps( @@ -1295,3 +1328,37 @@ def test_given_run_older_than_retention_when_history_is_missing_then_status_is_u assert summary.status == test_case.expected_status assert summary.query_count == test_case.expected_query_count assert "no longer available" in (summary.message or "") + + +@pytest.mark.parametrize( + "test_case", + [ + HistoryWindowTestCase( + description="a run that just finished reads open-ended history for running queries", + completed_minutes_ago=0, + expected_bounded=False, + ), + HistoryWindowTestCase( + description="an older run reads only its own window, not the hours since", + completed_minutes_ago=30, + expected_bounded=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_run_age_when_reading_history_then_older_runs_read_a_bounded_window( + test_case: HistoryWindowTestCase, +) -> None: + completed_at: datetime = datetime.now(UTC) - timedelta(minutes=test_case.completed_minutes_ago) + connection: _Connection = _Connection(()) + + snowflake_query_observations( + connection=connection, + started_at=completed_at - timedelta(minutes=1), + completed_at=completed_at, + expected_query_ids=frozenset(), + classification_query_ids=frozenset(), + deadline=float("inf"), + ) + + assert ("END_TIME_RANGE_END" in connection.sql) is test_case.expected_bounded