From 72f1220ca65d9810af678889f4da8ea9af35f63f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:31:32 +0000 Subject: [PATCH 01/11] fix!: turn silent fallbacks into clear errors (work in progress) Cursor delete_insert with a unique_key also replaces rows whose key is in the delta. Python loaders type Decimal values as DECIMAL and reject unrecognised types. keep_in_use stops when usage can't be read in full. Databricks freshness observes each source on its own. Project-level target locations resolve like models' locations. Not yet reviewed; make check-ci not yet run. --- benchmarks/duckshop/README.md | 2 +- benchmarks/duckshop/RUNBOOK.md | 14 +- .../int_edge_delivery_status_replaced.sql | 46 +++++++ .../src/janitor_text/_helpers/usage.rs | 12 +- .../src/janitor_text/constants.rs | 2 - .../src/janitor_text/tests/helpers.rs | 3 +- .../tests/test_janitor_usage_text.rs | 27 +--- .../_helpers/compile_attachment/target.rs | 58 ++++++-- .../compile_driver/assembly_project.rs | 50 ++++--- .../_helpers/compile_driver/targets.rs | 13 +- .../_helpers/janitor/janitor_usage.rs | 45 ++++++- .../planner_strategy/model_entries.rs | 7 + .../adapter/contract/classes/base_adapter.py | 16 ++- .../contract/classes/duckdb_backed_adapter.py | 16 ++- .../contract/classes/materialization.py | 1 + .../contract/classes/strict_adapter.py | 7 +- src/sqlbuild/adapter/contract/types.py | 1 + .../bigquery/classes/bigquery_adapter.py | 21 ++- src/sqlbuild/adapters/bigquery/constants.py | 2 + .../clickhouse/classes/clickhouse_adapter.py | 17 ++- .../databricks/classes/databricks_adapter.py | 123 ++++++----------- .../motherduck/classes/motherduck_adapter.py | 6 +- .../postgres/classes/postgres_adapter.py | 16 ++- .../snowflake/classes/snowflake_adapter.py | 16 ++- .../adapters/sqlite/classes/sqlite_adapter.py | 16 ++- .../sqlserver/classes/sqlserver_adapter.py | 16 ++- src/sqlbuild/executor/load/_helpers/rows.py | 109 +++++++++++++-- .../executor/load/_helpers/staging.py | 11 +- src/sqlbuild/executor/load/models.py | 26 ++++ .../_helpers/materializations/incremental.py | 12 ++ .../test_clickhouse_janitor_usage.py | 26 ++-- .../cli/commands/main/debug/test_debug.py | 39 ++++++ .../cli/commands/main/janitor/_test_types.py | 11 ++ .../cli/commands/main/janitor/helpers.py | 6 +- .../main/janitor/test_janitor_usage.py | 64 +++++++-- .../sqlbuild/adapters/clickhouse/helpers.py | 36 +++++ .../adapters/clickhouse/test_client.py | 16 +++ .../src/sqlbuild/adapters/sqlite/helpers.py | 15 +++ .../sqlbuild/adapters/sqlite/test_client.py | 15 +++ .../cli/commands/main/test_load_command.py | 125 ++++++++++++++++++ .../sqlbuild/compiler/pipeline/test_main.py | 19 ++- .../executor/build/test_build_execution.py | 41 ++++++ .../sqlbuild/adapter/contract/_test_types.py | 9 ++ .../adapter/contract/test_loader_types.py | 69 +++++++++- .../adapters/databricks/_test_types.py | 7 +- .../adapters/databricks/test_client.py | 69 +++++----- .../compile/_helpers/test_assembly.py | 34 +++++ .../sqlbuild/executor/janitor/main/helpers.py | 1 + .../src/sqlbuild/executor/load/test_rows.py | 2 +- website/src/content/docs/docs/cli/janitor.mdx | 14 +- .../src/content/docs/docs/concepts/diff.mdx | 4 + .../docs/docs/concepts/incremental.mdx | 18 ++- .../docs/concepts/models/configuration.mdx | 2 +- .../docs/concepts/project-configuration.mdx | 13 +- .../docs/concepts/python-nodes/loaders.mdx | 28 ++++ 55 files changed, 1129 insertions(+), 265 deletions(-) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_replaced.sql diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 4809449ba4..cdaa718f59 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -136,7 +136,7 @@ function, a hook-created JavaScript function, 192 unit tests and 4 scenarios, in | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | | Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; relations aliased `first` and `last`, one without `AS`, the words of Snowflake's read-order clauses; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | -| Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | +| Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild, and a keyed cursor `delete_insert` copy of it whose rows move out of the window between loads | | Translation refusals | one model per construct that translation tools refuse or guard, most putting a construct refused only under strict checking before one refused by default, so a model shows both: `FLOAT` values through sums, averages, running totals, arrays, objects, `HASH`, `TO_CHAR` and `::NUMBER(38, 10)`; `VARIANT` dates, decimals and XML; `ARRAY_AGG` without `WITHIN GROUP`, sparse arrays and `ARRAY(NUMBER)` casts; `REDUCE`, `FILTER` and `TRANSFORM` lambdas; patterns and `REGEXP_REPLACE` replacements built from columns, and lazy patterns; `TRY_TO_DATE` with a format from a column; zoned literals, `CONVERT_TIMEZONE` from `CURRENT_TIMESTAMP`, `SEQ8`, `GENERATOR` with an expression `ROWCOUNT`; `PIVOT`, `ANY_VALUE`, `IGNORE NULLS` around `IFF`, unaliased columns read by Snowflake's name, `UNION` branches whose column is always NULL; `TO_NUMBER`, `TO_DECIMAL` and `TRY_BASE64_DECODE_BINARY`; an open (uncontracted) source `edge__supplier_notes`, the table function, read for one basket and after `LATERAL` for every basket, and the hook-created function. `rpt_edge_case_model_keys` and `rpt_edge_case_array_model_keys` read every model in the group, so a refusal there also refuses a downstream model | Unit-test expectations in the edge-case area are captured on Snowflake under the `edge` target's diff --git a/benchmarks/duckshop/RUNBOOK.md b/benchmarks/duckshop/RUNBOOK.md index c766ae9abe..4ab8094084 100644 --- a/benchmarks/duckshop/RUNBOOK.md +++ b/benchmarks/duckshop/RUNBOOK.md @@ -196,21 +196,27 @@ every token being distinct. `int_edge_delivery_status` merges a shipment delivery change log incrementally, and the `edge_delivery_status_matches_rebuild` audit fails if it differs from a rebuild from the whole log. -To exercise the merge across loads, start without the table, load the first load only, build, then -load everything and build again: +`int_edge_delivery_status_replaced` copies it with a keyed cursor `delete_insert`, and its `unique` +audit fails if a shipment's earlier row survives. To exercise both across loads, start without the +tables, load the first load only, build, then load everything and build again: ```bash sqb query --target edge "DROP TABLE IF EXISTS ..int_edge_delivery_status" +sqb query --target edge "DROP TABLE IF EXISTS ..int_edge_delivery_status_replaced" sqb load --target edge --select edge__delivery_updates --vars '{"edge_delivery_batch": "1"}' sqb build --target edge --no-load \ - --select int_edge_delivery_update_versions int_edge_delivery_status int_edge_delivery_status_rebuilt + --select int_edge_delivery_update_versions int_edge_delivery_status int_edge_delivery_status_rebuilt \ + int_edge_delivery_status_replaced sqb load --target edge --select edge__delivery_updates sqb build --target edge --no-load \ - --select int_edge_delivery_update_versions int_edge_delivery_status int_edge_delivery_status_rebuilt + --select int_edge_delivery_update_versions int_edge_delivery_status int_edge_delivery_status_rebuilt \ + int_edge_delivery_status_replaced ``` The second build plans `int_edge_delivery_status` as a `merge` over the second load, which carries a late older version, a redelivered version, a delete and a new shipment, and the audit passes. +It plans `int_edge_delivery_status_replaced` as a `delete_insert` that replaces each recomputed +shipment by key, and its audits pass. Load with `sqb load` and build with `--no-load`, so that each build reads exactly the load you chose. diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_replaced.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_replaced.sql new file mode 100644 index 0000000000..e203dbd926 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_status_replaced.sql @@ -0,0 +1,46 @@ +MODEL ( + description """ + One row per shipment, copied from the merged delivery status by delete and insert. A shipment + with a later version moves its source_loaded_at out of the window an earlier build wrote, so + only its unique_key stops the later build from adding a second row for it. + """, + tags [edge_case], + materialized incremental, + incremental_strategy delete_insert, + unique_key [shipment_id], + cursor source_loaded_at, + cursor_type timestamp, + cursor_grain second, + cursor_inputs ( + int_edge_delivery_status source_loaded_at, + ), + columns ( + shipment_id (type VARCHAR, nullable false, audits [not_null, unique]), + order_ref (type VARCHAR), + delivery_status (type VARCHAR), + ordered_on (type DATE), + delivered_on (type DATE), + is_deleted (type BOOLEAN), + latest_seq (type NUMBER(38, 0)), + source_loaded_at (type TIMESTAMP_NTZ, nullable false, audits [not_null]), + ), +); + +WITH final AS ( + SELECT + s.shipment_id, + s.order_ref, + s.delivery_status, + s.ordered_on, + s.delivered_on, + s.is_deleted, + s.latest_seq, + s.source_loaded_at + FROM __ref("int_edge_delivery_status") AS s + WHERE + s.source_loaded_at >= __cursor_start() + AND s.source_loaded_at < __cursor_end() +) + +SELECT * +FROM final diff --git a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs index 5ea862ca81..5bac5de87c 100644 --- a/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs +++ b/crates/sqlbuild-core/src/janitor_text/_helpers/usage.rs @@ -2,9 +2,7 @@ use crate::cli_text::main::format_count_noun::format_count_noun; use crate::janitor_text::_helpers::phases::noun; -use crate::janitor_text::constants::{ - KEEP_WITHOUT_USAGE_NOTICE, READER_LIMIT, USAGE_NOT_CHECKED_PREFIX, WINDOW_UNITS, -}; +use crate::janitor_text::constants::{READER_LIMIT, USAGE_NOT_CHECKED_PREFIX, WINDOW_UNITS}; use crate::janitor_text::models::{ArchiveCandidateText, JanitorPlanText, UsageCheckText}; /// The usage check when it ran and could read usage. @@ -81,14 +79,10 @@ pub(crate) fn usage_notices(plan: &JanitorPlanText) -> Vec { if check.checked { return check.notices.clone(); } - let mut notices: Vec = vec![format!( + vec![format!( "{USAGE_NOT_CHECKED_PREFIX}{}", check.reason.as_deref().unwrap_or("unknown reason") - )]; - if check.keep_in_use { - notices.push(KEEP_WITHOUT_USAGE_NOTICE.to_owned()); - } - notices + )] } /// The prompt line stating how many archived relations are in use, or which are kept. diff --git a/crates/sqlbuild-core/src/janitor_text/constants.rs b/crates/sqlbuild-core/src/janitor_text/constants.rs index 148a5f3ee0..e4b4550295 100644 --- a/crates/sqlbuild-core/src/janitor_text/constants.rs +++ b/crates/sqlbuild-core/src/janitor_text/constants.rs @@ -25,8 +25,6 @@ pub(crate) const SECONDS_PER_HOUR: u64 = 3600; /// How many readers each in-use line names before `+N more`. pub(crate) const READER_LIMIT: usize = 3; pub(crate) const USAGE_NOT_CHECKED_PREFIX: &str = "usage not checked: "; -pub(crate) const KEEP_WITHOUT_USAGE_NOTICE: &str = - "nothing kept in use: no usage could be checked, so relations in use are archived too"; /// Single-unit usage windows and the noun of their unit. pub(crate) const WINDOW_UNITS: [(&str, &str); 4] = [ ("d", "day"), diff --git a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs index a1d63fe2c1..88fb730602 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/helpers.rs @@ -202,13 +202,12 @@ pub(super) fn kept_usage_plan() -> JanitorPlanText { } /// A plan whose usage was not checked because the adapter has no usage source. -pub(super) fn unchecked_plan(keep_in_use: bool) -> JanitorPlanText { +pub(super) fn unchecked_plan() -> JanitorPlanText { JanitorPlanText { usage_check: Some(UsageCheckText { checked: false, reason: Some("unsupported adapter".to_owned()), window: "7d".to_owned(), - keep_in_use, ..UsageCheckText::default() }), ..direct_archive_plan() diff --git a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs index 23a2260f8a..88615c78f7 100644 --- a/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/tests/test_janitor_usage_text.rs @@ -66,24 +66,14 @@ fn given_usage_check_when_rendering_preview_then_in_use_relations_show_readers() "\n view chains not followed: grant SNOWFLAKE.OBJECT_VIEWER\n", "\nIn use\n", ], - unexpected_fragments: &["usage not checked", "nothing kept in use"], + unexpected_fragments: &["usage not checked"], }, PreviewTestCase { description: "an unchecked usage shows one notice and no in-use rows", - plan: unchecked_plan(false), + plan: unchecked_plan(), verbose: false, expected_fragments: &["\n usage not checked: unsupported adapter\n"], - unexpected_fragments: &["in use ", "In use", "nothing kept in use"], - }, - PreviewTestCase { - description: "keep_in_use without a usage source warns that nothing is kept", - plan: unchecked_plan(true), - verbose: false, - expected_fragments: &[ - " usage not checked: unsupported adapter\n nothing kept in use: no usage \ - could be checked, so relations in use are archived too\n", - ], - unexpected_fragments: &["Kept (in use)"], + unexpected_fragments: &["in use ", "In use"], }, PreviewTestCase { description: "a plan without a usage check renders as before", @@ -141,18 +131,11 @@ fn given_usage_check_when_prompting_then_prompt_states_in_use_count() { }, UsagePromptTestCase { description: "an unchecked usage repeats the notice", - plan: unchecked_plan(false), + plan: unchecked_plan(), expected_fragment: "Janitor will archive 1 and delete 1 objects from dev.\n\ Usage not checked: unsupported adapter\nRetention: 14 days\n", expected_phrase: "archive 1 and delete 1 objects from dev", }, - UsagePromptTestCase { - description: "keep_in_use without a usage source repeats the warning", - plan: unchecked_plan(true), - expected_fragment: "Usage not checked: unsupported adapter\nNothing kept in use: \ - no usage could be checked, so relations in use are archived too\nRetention", - expected_phrase: "archive 1 and delete 1 objects from dev", - }, ]; for test_case in test_cases { @@ -214,7 +197,7 @@ fn given_usage_check_when_formatting_json_then_candidates_carry_in_use() { }, JsonTestCase { description: "an unchecked usage leaves in_use null and states the reason", - plan: Some(unchecked_plan(false)), + plan: Some(unchecked_plan()), outcome: JanitorOutcome::Previewed, run: None, error: None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 0a536f34de..9cbe3f8877 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -3,12 +3,15 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::compile_attachment::effective_config::project_vars; use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_CACHE_DIRECTORY_NAMES, }; +use crate::bindings::_helpers::compile_driver::targets::requires_explicit_targets; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; @@ -121,16 +124,27 @@ fn optional_string<'py>( }) } -/// `first or second or third`. -fn first_truthy<'py>(values: [Bound<'py, PyAny>; 3]) -> PyResult> { - let [first, second, third] = values; - if first.is_truthy()? { - return Ok(first); +/// A project-level location as models resolve it: target, then `[defaults]`, then any fallback. +pub(crate) fn project_location<'py>( + (target, default): (Bound<'py, PyAny>, Bound<'py, PyAny>), + fallbacks: Vec>, + explicit: bool, +) -> PyResult> { + let py: Python<'py> = target.py(); + if target.is_truthy()? && !target.eq(PRESERVE_TARGET_VALUE)? { + return Ok(target); + } + if default.is_truthy()? { + return Ok(default); } - if second.is_truthy()? { - return Ok(second); + if !explicit { + for fallback in fallbacks { + if fallback.is_truthy()? { + return Ok(fallback); + } + } } - Ok(third) + Ok(py.None().into_bound(py)) } /// Resolve the target and its effective database/schema without compiling resources. @@ -170,18 +184,34 @@ pub(crate) fn build_effective_target_namespace<'py>( } native_field(&target_config, key) }; + let project_config: Bound<'py, PyAny> = native_field(discovered_inputs, "project_config")?; + let defaults: Bound<'py, PyAny> = native_field(&project_config, "defaults")?; + let explicit: bool = requires_explicit_targets(&resolve_effective_adapter_name( + &project_config, + &native_field(discovered_inputs, "local_config")?, + )?)?; let values: Bound<'py, PyDict> = PyDict::new(py); values.set_item( DATABASE_KEY, - first_truthy([ - target_value(DATABASE_KEY)?, - connection_database, - default_database, - ])?, + project_location( + ( + target_value(DATABASE_KEY)?, + native_field(&defaults, DATABASE_KEY)?, + ), + vec![connection_database, default_database], + explicit, + )?, )?; values.set_item( SCHEMA_KEY, - first_truthy([target_value(SCHEMA_KEY)?, connection_schema, default_schema])?, + project_location( + ( + target_value(SCHEMA_KEY)?, + native_field(&defaults, SCHEMA_KEY)?, + ), + vec![connection_schema, default_schema], + explicit, + )?, )?; let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; let resolved: Bound<'py, PyDict> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index d847c9317d..faf1c2f1d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -10,6 +10,7 @@ use sqlbuild_analysis::assembly::project::main::configured_namespace::configured use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compile_attachment::target::project_location; use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, analysis_context, and_not, @@ -23,6 +24,7 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compile_driver::stage_work::{SEMANTIC_COMPLETION, count_stage}; +use crate::bindings::_helpers::compile_driver::targets::requires_explicit_targets; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::contracts::spec_resolution::{ resolve_effective_adapter_name, resolve_effective_scenario_config, @@ -171,22 +173,29 @@ fn target_field<'py>(inputs: &Bound<'py, PyAny>, field: &str) -> PyResult(inputs: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = inputs.py(); let connection: Bound<'py, PyAny> = native_field(inputs, "effective_connection")?; - let database: Bound<'py, PyAny> = target_field(inputs, "database")?; - let database: Bound<'py, PyAny> = if database.is_truthy()? { - database - } else { - connection_database_fallback(inputs, &connection)? - }; - let schema: Bound<'py, PyAny> = target_field(inputs, "schema")?; - let schema: Bound<'py, PyAny> = if schema.is_truthy()? { - schema - } else { - str_or_none(&mapping_get(&connection, "schema")?)? - }; + let defaults: Bound<'py, PyAny> = + native_field(&native_field(inputs, "project_config")?, "defaults")?; + let explicit: bool = requires_explicit_targets(&effective_adapter_name(inputs)?)?; + let database: Bound<'py, PyAny> = project_location( + ( + target_field(inputs, "database")?, + native_field(&defaults, "database")?, + ), + vec![connection_database_fallback(inputs, &connection)?], + explicit, + )?; + let schema: Bound<'py, PyAny> = project_location( + ( + target_field(inputs, "schema")?, + native_field(&defaults, "schema")?, + ), + vec![str_or_none(&mapping_get(&connection, "schema")?)?], + explicit, + )?; early_model_templates( keywords(py, &[("database", database), ("schema", schema)])?.as_any(), &native_field(inputs, "effective_vars")?, @@ -197,17 +206,22 @@ fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult(inputs: &Bound<'py, PyAny>) -> PyResult> { + resolve_effective_adapter_name( + &native_field(inputs, "project_config")?, + &native_field(inputs, "local_config")?, + ) + .and_then(|value| value.into_bound_py_any(inputs.py())) +} + /// The connection's database, except under DuckDB, whose database is a file. fn connection_database_fallback<'py>( inputs: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( - &native_field(inputs, "project_config")?, - &native_field(inputs, "local_config")?, - ) - .and_then(|value| value.into_bound_py_any(py))?; + let adapter_name: Bound<'py, PyAny> = effective_adapter_name(inputs)?; if adapter_name.eq(host::entry(py, AdapterHost::BuiltinAdapter)?.getattr("DUCKDB")?)? { return Ok(py.None().into_bound(py)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 41b587e0f9..9171a9aafd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -390,16 +390,19 @@ fn validate_project_targets<'py>( ) } +/// Whether the adapter needs every model and seed database and schema resolved. +pub(crate) fn requires_explicit_targets(adapter_name: &Bound<'_, PyAny>) -> PyResult { + Ok(text_or_none(adapter_name)? + .as_deref() + .is_some_and(|name| EXPLICIT_TARGET_ADAPTERS.contains(&name))) +} + /// `validate_project_targets` with its arguments. pub(crate) fn project_targets( adapter_name: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult<()> { - let name: Option = text_or_none(adapter_name)?; - if !name - .as_deref() - .is_some_and(|name| EXPLICIT_TARGET_ADAPTERS.contains(&name)) - { + if !requires_explicit_targets(adapter_name)? { return Ok(()); } let adapter_label: String = adapter_name.str()?.to_cow()?.into_owned(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs index bf66ebde19..eb8d80f94c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_usage.rs @@ -4,19 +4,26 @@ use std::collections::{HashMap, HashSet}; use std::time::Instant; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyString}; +use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; use crate::bindings::_helpers::boundary::python_calls::{call_with, within}; use crate::bindings::_helpers::build_progress::transient_status::TransientStatusReporter; +use crate::bindings::_helpers::diff_report::support::cli_error; use crate::bindings::_helpers::janitor::janitor_keys::{Folded, Key, record, tuple}; use crate::bindings::_helpers::janitor::janitor_planning::completed; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::ErrorId; const START_MESSAGE: &str = "Checking relation usage..."; +const INCOMPLETE_USAGE_CODE: &str = "C503"; +const INCOMPLETE_USAGE_HELP: &str = "grant the role read access to the warehouse's query history \ + (see `sqb janitor`, \"Usage comes from the warehouse's query history\"), or run without \ + keep_in_use to archive relations whether or not they are read"; /// The usage window and whether in-use candidates are kept. pub(crate) struct UsageSettings { @@ -96,6 +103,27 @@ fn completion_message( format!("Checked relation usage: {in_use} of {total} in use. ({elapsed_seconds:.2}s)") } +/// Why `keep_in_use` cannot run: usage was not checked, or the check left something out. +fn incomplete_usage_error(py: Python<'_>, reason: Option, notices: &[String]) -> PyErr { + let missing: String = match reason { + Some(reason) if notices.is_empty() => format!("usage was not checked: {reason}"), + None if notices.is_empty() => "usage was not checked".to_owned(), + _ => format!("usage was only partly checked: {}", notices.join("; ")), + }; + cli_error( + py, + ErrorId::CliUser, + &DiffUserError { + message: format!( + "keep_in_use needs the usage of every relation to archive, but {missing}; \ + nothing was archived" + ), + code: INCOMPLETE_USAGE_CODE.to_owned(), + help: Some(INCOMPLETE_USAGE_HELP.to_owned()), + }, + ) +} + /// The plan with each candidate's reads, the usage check, and in-use candidates kept when asked. pub(crate) fn check_janitor_usage<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -127,8 +155,20 @@ pub(crate) fn check_janitor_usage<'py>( } }; let checked: bool = native_field(&report, "checked")?.is_truthy()?; + let reason: Bound<'py, PyAny> = native_field(&report, "reason")?; + let notices: Vec = native_field(&report, "notices")?.extract()?; + if settings.keep_in_use && (!checked || !notices.is_empty()) { + TransientStatusReporter::error( + status, + &format!( + "Relation usage check incomplete. ({:.2}s)", + started.elapsed().as_secs_f64() + ), + )?; + return Err(incomplete_usage_error(py, reason.extract()?, ¬ices)); + } let usages: HashMap> = usages_by_relation(&report)?; - let keep: bool = checked && settings.keep_in_use; + let keep: bool = settings.keep_in_use; let mut archived: Vec> = Vec::new(); let mut kept: Vec> = Vec::new(); let mut kept_keys: HashSet = HashSet::new(); @@ -159,7 +199,6 @@ pub(crate) fn check_janitor_usage<'py>( deletions.push(deletion); } } - let reason: Bound<'py, PyAny> = native_field(&report, "reason")?; TransientStatusReporter::complete( status, &completion_message( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index e5c4ca0de3..bd6b2d9c01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -279,6 +279,13 @@ fn logical_ddl<'py>( "cursor_type", or_none(py, config_text_object(py, &values, "cursor_type")?), )); + let microbatch: bool = match config_text_object(py, &values, "incremental_mode")? { + Some(mode) => mode.eq(MICROBATCH_MODE)?, + None => false, + }; + if !microbatch { + pairs.push(("unique_key", unique_key.clone().into_any())); + } "render_delete_insert_cursor" } "incremental_delete_insert" => { diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 847c7a2c65..e1d7cb17e8 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -710,6 +710,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: delete_sql: str = ( f"DELETE FROM {destination} " @@ -717,6 +718,13 @@ def render_delete_insert_cursor( f"{self._quote_sql_string(cursor_start)} " f"AND {self.render_identifier(cursor_column)} < {self._quote_sql_string(cursor_end)}" ) + if unique_key: + return ( + delete_sql, + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) insert_stmts: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) @@ -1488,6 +1496,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> int | None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, @@ -1497,6 +1506,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) affected: int | None = None @@ -1976,7 +1986,9 @@ def render_framework_type(self, type_name: FrameworkType) -> str: case FrameworkType.TIMESTAMP: return "TIMESTAMP" - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: """Render one source-loader logical type using generic SQL defaults.""" match type_name: @@ -1986,6 +1998,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "BIGINT" case LoaderLogicalType.FLOAT: return "DOUBLE" + case LoaderLogicalType.DECIMAL: + return f"DECIMAL({precision}, {scale})" case LoaderLogicalType.STRING: return "VARCHAR" case LoaderLogicalType.TIMESTAMP: diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 95a55403ae..476b47d91e 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -414,7 +414,9 @@ def render_exact_identifier(self, name: str) -> str: return self.render_identifier(name) - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BOOLEAN" @@ -422,6 +424,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "BIGINT" case LoaderLogicalType.FLOAT: return "DOUBLE" + case LoaderLogicalType.DECIMAL: + return f"DECIMAL({precision}, {scale})" case LoaderLogicalType.STRING: return "VARCHAR" case LoaderLogicalType.TIMESTAMP: @@ -1603,6 +1607,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: delete_sql: str = ( f"DELETE FROM {destination} " @@ -1610,6 +1615,13 @@ def render_delete_insert_cursor( f"{self._quote_sql_string(cursor_start)} " f"AND {self.render_identifier(cursor_column)} < {self._quote_sql_string(cursor_end)}" ) + if unique_key: + return ( + delete_sql, + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) insert_stmts: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) @@ -2050,6 +2062,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> int | None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, @@ -2059,6 +2072,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) affected: int | None = None diff --git a/src/sqlbuild/adapter/contract/classes/materialization.py b/src/sqlbuild/adapter/contract/classes/materialization.py index 6d86984026..e36ab176d8 100644 --- a/src/sqlbuild/adapter/contract/classes/materialization.py +++ b/src/sqlbuild/adapter/contract/classes/materialization.py @@ -170,6 +170,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> int | None: """Delete rows by cursor range then insert from a SELECT statement.""" ... diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 4f8ec2cb57..19023aa933 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -388,6 +388,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: """Render SQL statements for cursor-bounded delete-insert.""" ... @@ -888,8 +889,10 @@ def render_framework_type(self, type_name: FrameworkType) -> str: ... @abstractmethod - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: - """Render one source-loader logical type for this adapter.""" + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: + """Render one source-loader logical type; DECIMAL uses precision and scale.""" ... @abstractmethod diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 6ed7dd4f6b..0ef3963263 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -103,6 +103,7 @@ class LoaderLogicalType(StrEnum): BOOLEAN = "boolean" INTEGER = "integer" FLOAT = "float" + DECIMAL = "decimal" STRING = "string" TIMESTAMP = "timestamp" DATE = "date" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 3aa5666d9a..6aac74eaff 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -142,6 +142,8 @@ INTEGER_WIRE_TYPE_NAME, NATIVE_TABLE_TYPE, NOT_FOUND_ERROR_CLASS_NAME, + NUMERIC_FRACTION_DIGITS, + NUMERIC_INTEGER_DIGITS, SELECT_STATEMENT_TYPE, STRING_LITERAL_ESCAPES, TABLE_NAME_WILDCARD, @@ -1080,7 +1082,9 @@ def render_current_timestamp(self) -> str: def render_current_utc_timestamp(self) -> str: return "CURRENT_TIMESTAMP()" - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BOOL" @@ -1088,6 +1092,10 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "INT64" case LoaderLogicalType.FLOAT: return "FLOAT64" + case LoaderLogicalType.DECIMAL: + if precision - scale <= NUMERIC_INTEGER_DIGITS and scale <= NUMERIC_FRACTION_DIGITS: + return f"NUMERIC({precision}, {scale})" + return f"BIGNUMERIC({precision}, {scale})" case LoaderLogicalType.STRING: return "STRING" case LoaderLogicalType.TIMESTAMP: @@ -1810,6 +1818,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: insert_clause: str = "INSERT ROW" if columns is not None: @@ -1824,6 +1833,14 @@ def render_delete_insert_cursor( f"AND __target.{self.render_identifier(cursor_column)} < " f"{self._render_cursor_bound_string(cursor_end)}" ) + if unique_key: + return ( + f"DELETE FROM {self._quote_identifier_path(destination)} AS __target " + f"WHERE {cursor_filter}", + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) return ( f"MERGE {self._quote_identifier_path(destination)} AS __target " f"USING ({sql}) AS __source ON FALSE " @@ -2522,6 +2539,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> int | None: if columns is None: columns = self.query_column_names(connection=connection, sql=sql) @@ -2533,6 +2551,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) affected: int | None = None diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index 925e79dc17..fb6fbf09c9 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -59,3 +59,5 @@ IDENTIFIER_ESCAPES: dict[int, str] = str.maketrans( {"\\": "\\\\", "`": "\\`", "\n": "\\n", "\r": "\\r"} ) +NUMERIC_INTEGER_DIGITS: int = 29 +NUMERIC_FRACTION_DIGITS: int = 9 diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index e4ef093412..74bd7d7ae7 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -529,7 +529,9 @@ def render_framework_type(self, type_name: FrameworkType) -> str: case FrameworkType.TIMESTAMP: return "Nullable(DateTime64(6, 'UTC'))" - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "Nullable(Bool)" @@ -537,6 +539,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "Nullable(Int64)" case LoaderLogicalType.FLOAT: return "Nullable(Float64)" + case LoaderLogicalType.DECIMAL: + return f"Nullable(Decimal({precision}, {scale}))" case LoaderLogicalType.STRING: return "Nullable(String)" case LoaderLogicalType.TIMESTAMP: @@ -965,6 +969,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: request: PartitionRewrite = self._window_rewrite( destination=destination, @@ -974,6 +979,7 @@ def render_delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) return ( request.delta_insert_sql, @@ -1032,6 +1038,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> int | None: return self._rewrite( connection=connection, @@ -1043,6 +1050,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ), statement_recorder=statement_recorder, ) @@ -1110,14 +1118,19 @@ def _window_rewrite( cursor_end: str, columns: tuple[str, ...] | None, cursor_type: str | None, + unique_key: tuple[str, ...], ) -> PartitionRewrite: quoted: str = self.render_identifier(cursor_column) start: str = self.render_cursor_bound_literal(value=cursor_start, cursor_type=cursor_type) end: str = self.render_cursor_bound_literal(value=cursor_end, cursor_type=cursor_type) + displaced: str = f"({quoted} >= {start} AND {quoted} < {end})" + if unique_key: + key_tuple: str = _key_tuple(adapter=self, keys=unique_key) + displaced = f"({displaced} OR {key_tuple} IN (SELECT {key_tuple} FROM {{delta}}))" return PartitionRewrite( destination=destination, delta_insert_sql=self.render_append(destination="{delta}", sql=sql, columns=columns)[0], - displaced_predicate=f"({quoted} >= {start} AND {quoted} < {end})", + displaced_predicate=displaced, ) def _merge_rewrite( diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 8914544982..8bda13a764 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -657,87 +657,13 @@ def get_tables_freshness_metadata( ) if not lookup_requests: return results - try: - results.update( - self._get_delta_history_freshness_metadata( - connection=connection, - requests=lookup_requests, - ) - ) - except Exception: - results.update( - self._get_unity_catalog_freshness_metadata( - connection=connection, - requests=lookup_requests, - ) - ) - return results - - def _get_unity_catalog_freshness_metadata( - self, - *, - connection: Any, - requests: tuple[TableFreshnessRequest, ...], - ) -> dict[TableFreshnessRequest, TableFreshnessMetadata]: - clauses: str = " OR ".join( - "(table_catalog = " - + self._string_literal(str(request.database).lower()) - + " AND table_schema = " - + self._string_literal(str(request.schema).lower()) - + " AND table_name = " - + self._string_literal(request.name.lower()) - + ")" - for request in requests - ) - query: str = ( - "SELECT table_catalog, table_schema, table_name, table_type, last_altered " - "FROM `system`.`information_schema`.`tables` WHERE " - f"{clauses}" - ) - cursor: Any = connection.cursor() - try: - cursor.execute(query) - rows: list[tuple[Any, ...]] = list(cursor.fetchall()) - finally: - cursor.close() - - requests_by_key: dict[tuple[str, str, str], TableFreshnessRequest] = { - ( - str(request.database).lower(), - str(request.schema).lower(), - request.name.lower(), - ): request - for request in requests - } - results: dict[TableFreshnessRequest, TableFreshnessMetadata] = {} - row: tuple[Any, ...] - for row in rows: - matched_request: TableFreshnessRequest | None = requests_by_key.get( - (str(row[0]).lower(), str(row[1]).lower(), str(row[2]).lower()) - ) - if matched_request is None: - continue - table_type: str = str(row[3]) - if self._normalize_relation_type(table_type) != RelationType.TABLE: - results[matched_request] = TableFreshnessMetadata.unavailable( - message="Databricks table freshness metadata only supports Delta tables; " - f"found {table_type} for {matched_request.name}" - ) - continue - if row[4] is None: - results[matched_request] = TableFreshnessMetadata.unavailable( - message="Databricks table freshness metadata is missing LAST_ALTERED " - f"for {matched_request.name}" - ) - continue - results[matched_request] = TableFreshnessMetadata( - data_version=row[4], - value_kind="timestamp", - observed_at=row[4] if isinstance(row[4], datetime) else None, + results.update( + self._get_delta_history_freshness_metadata( + connection=connection, + requests=lookup_requests, ) - return complete_table_freshness_results( - requests=requests, results=results, adapter_label="Databricks" ) + return results def _get_delta_history_freshness_metadata( self, @@ -772,10 +698,22 @@ def _get_delta_history_freshness_metadata( cursor.execute(" UNION ALL ".join(selects)) rows: list[tuple[Any, ...]] = list(cursor.fetchall()) except Exception as error: - raise AdapterUserError( - message="Databricks table freshness metadata requires Unity Catalog table metadata " - "or Delta history" - ) from error + if len(requests) == 1: + reason: str = (str(error).strip().splitlines() or [type(error).__name__])[0] + return { + requests[0]: TableFreshnessMetadata.unavailable( + message=f"Databricks Delta history unavailable for {requests[0].name}: " + f"{reason}" + ) + } + results: dict[TableFreshnessRequest, TableFreshnessMetadata] = {} + for request in requests: + results.update( + self._get_delta_history_freshness_metadata( + connection=connection, requests=(request,) + ) + ) + return results finally: cursor.close() @@ -792,9 +730,10 @@ def _get_delta_history_freshness_metadata( if matched_request is None: continue if row[3] is None: - raise AdapterUserError( + results[matched_request] = TableFreshnessMetadata.unavailable( message=f"Databricks Delta history not found for {matched_request.name}" ) + continue results[matched_request] = TableFreshnessMetadata( data_version=row[3], value_kind="timestamp", @@ -878,7 +817,9 @@ def render_identifier(self, name: str) -> str: def render_exact_identifier(self, name: str) -> str: return self.render_identifier(name) - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BOOLEAN" @@ -886,6 +827,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "BIGINT" case LoaderLogicalType.FLOAT: return "DOUBLE" + case LoaderLogicalType.DECIMAL: + return f"DECIMAL({precision}, {scale})" case LoaderLogicalType.STRING: return "STRING" case LoaderLogicalType.TIMESTAMP: @@ -1864,6 +1807,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: column_list: str = "" if columns is not None: @@ -1874,6 +1818,13 @@ def render_delete_insert_cursor( replace_where: str = ( f"{quoted_cursor_column} >= {start_bound} AND {quoted_cursor_column} < {end_bound}" ) + if unique_key: + return ( + f"DELETE FROM {destination} WHERE {replace_where}", + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) return (f"INSERT INTO {destination}{column_list} REPLACE WHERE {replace_where} {sql}",) def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: @@ -2321,6 +2272,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, @@ -2330,6 +2282,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) statement: str diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index d14283f6b6..468d32013a 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -32,7 +32,9 @@ class MotherDuckAdapter( adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value sql_analysis_dialect_name: ClassVar[str | None] = "duckdb" - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BOOLEAN" @@ -40,6 +42,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "BIGINT" case LoaderLogicalType.FLOAT: return "DOUBLE" + case LoaderLogicalType.DECIMAL: + return f"DECIMAL({precision}, {scale})" case LoaderLogicalType.STRING: return "VARCHAR" case LoaderLogicalType.TIMESTAMP: diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 581f912dbd..321e6d566d 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -640,6 +640,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: quoted_cursor_column: str = self.render_identifier(cursor_column) delete_sql: str = ( @@ -647,6 +648,13 @@ def render_delete_insert_cursor( f"{self._quote_sql_string(cursor_start)} " f"AND {quoted_cursor_column} < {self._quote_sql_string(cursor_end)}" ) + if unique_key: + return ( + delete_sql, + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) insert_stmts: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) @@ -1278,6 +1286,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, @@ -1287,6 +1296,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) with self.transaction(connection): @@ -1967,7 +1977,9 @@ def render_exact_identifier(self, name: str) -> str: return self.render_identifier(name) - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BOOLEAN" @@ -1975,6 +1987,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "BIGINT" case LoaderLogicalType.FLOAT: return "DOUBLE PRECISION" + case LoaderLogicalType.DECIMAL: + return f"NUMERIC({precision}, {scale})" case LoaderLogicalType.STRING: return "TEXT" case LoaderLogicalType.TIMESTAMP: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 6ba1c3a986..22bfa43afa 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -1191,6 +1191,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: quoted_cursor_column: str = self.render_identifier(cursor_column) delete_sql: str = ( @@ -1198,6 +1199,13 @@ def render_delete_insert_cursor( f"{self._quote_sql_string(cursor_start)} " f"AND {quoted_cursor_column} < {self._quote_sql_string(cursor_end)}" ) + if unique_key: + return ( + delete_sql, + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) insert_stmts: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) @@ -1573,6 +1581,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> int | None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, @@ -1582,6 +1591,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) with self.transaction(connection): @@ -1625,7 +1635,9 @@ def render_exact_identifier(self, name: str) -> str: return '"' + name.replace('"', '""') + '"' - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BOOLEAN" @@ -1633,6 +1645,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "BIGINT" case LoaderLogicalType.FLOAT: return "DOUBLE" + case LoaderLogicalType.DECIMAL: + return f"NUMBER({precision}, {scale})" case LoaderLogicalType.STRING: return "VARCHAR" case LoaderLogicalType.TIMESTAMP: diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index b933c09664..d28382af60 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -719,6 +719,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: quoted_cursor_column: str = self.render_cursor_operand( expression=self.render_identifier(cursor_column), cursor_type=cursor_type @@ -733,6 +734,13 @@ def render_delete_insert_cursor( f"DELETE FROM {destination} WHERE {quoted_cursor_column} >= {start_literal} " f"AND {quoted_cursor_column} < {end_literal}" ) + if unique_key: + return ( + delete_sql, + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) insert_stmts: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) @@ -1262,6 +1270,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> int | None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, @@ -1271,6 +1280,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) affected: int | None = None @@ -2237,7 +2247,9 @@ def render_framework_type(self, type_name: FrameworkType) -> str: case FrameworkType.TIMESTAMP: return "TIMESTAMP" - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BOOLEAN" @@ -2245,6 +2257,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "INTEGER" case LoaderLogicalType.FLOAT: return "REAL" + case LoaderLogicalType.DECIMAL: + return f"DECIMAL({precision}, {scale})" case LoaderLogicalType.STRING: return "TEXT" case LoaderLogicalType.TIMESTAMP: diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index fe336d2821..c279286f0d 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -970,7 +970,9 @@ def render_framework_type(self, type_name: FrameworkType) -> str: case FrameworkType.TIMESTAMP: return "DATETIME2" - def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + def render_loader_logical_type( + self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 + ) -> str: match type_name: case LoaderLogicalType.BOOLEAN: return "BIT" @@ -978,6 +980,8 @@ def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: return "BIGINT" case LoaderLogicalType.FLOAT: return "FLOAT" + case LoaderLogicalType.DECIMAL: + return f"DECIMAL({precision}, {scale})" case LoaderLogicalType.STRING: return "NVARCHAR(MAX)" case LoaderLogicalType.TIMESTAMP: @@ -1351,6 +1355,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> None: statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, @@ -1360,6 +1365,7 @@ def delete_insert_cursor( cursor_end=cursor_end, columns=columns, cursor_type=cursor_type, + unique_key=unique_key, ) statement_recorder.record_many(statements) with self.transaction(connection): @@ -2310,6 +2316,7 @@ def render_delete_insert_cursor( cursor_end: str, columns: tuple[str, ...] | None = None, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: start_literal: str = self.render_cursor_bound_literal( value=cursor_start, cursor_type=cursor_type @@ -2322,6 +2329,13 @@ def render_delete_insert_cursor( f"WHERE {self.render_identifier(cursor_column)} >= {start_literal} " f"AND {self.render_identifier(cursor_column)} < {end_literal}" ) + if unique_key: + return ( + delete_sql, + *self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ), + ) insert_stmts: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) diff --git a/src/sqlbuild/executor/load/_helpers/rows.py b/src/sqlbuild/executor/load/_helpers/rows.py index 54009795c9..113f72b67f 100644 --- a/src/sqlbuild/executor/load/_helpers/rows.py +++ b/src/sqlbuild/executor/load/_helpers/rows.py @@ -5,13 +5,16 @@ from collections.abc import Iterable, Iterator, Mapping from datetime import date, datetime from decimal import Decimal +from types import MappingProxyType from sqlbuild._native import SourceColumnEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import LoaderLogicalType from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.load.models import LoaderRowsSchema +from sqlbuild.executor.load.models import DecimalShape, LoaderRowsSchema + +_MAX_DECIMAL_PRECISION: int = 38 def iter_loader_row_batches( @@ -51,13 +54,17 @@ def update_loader_rows_schema( columns: tuple[SourceColumnEntry, ...], column_names: tuple[str, ...], inferred_types: dict[str, LoaderLogicalType], + decimal_shapes: Mapping[str, DecimalShape] = MappingProxyType({}), contract_enforced: bool = False, ) -> LoaderRowsSchema: - """Merge one loader row batch into the tracked staging schema.""" + """Merge one loader row batch into the tracked staging schema, keeping each decimal's width.""" declared_types: dict[str, str] = _declared_types(columns=columns) names: list[str] = list(column_names) next_inferred_types: dict[str, LoaderLogicalType] = dict(inferred_types) + batch_shapes: dict[str, DecimalShape] = _decimal_shapes( + rows=rows, declared_types=declared_types + ) added_column_names: list[str] = [] row: dict[str, object] for row in rows: @@ -74,7 +81,9 @@ def update_loader_rows_schema( added_column_names.append(column_name) if column_name in declared_types: continue - logical_type: LoaderLogicalType | None = _infer_value_type(value) + logical_type: LoaderLogicalType | None = _infer_value_type( + column_name=column_name, value=value + ) if logical_type is None and column_name in added_column_names: logical_type = LoaderLogicalType.STRING if logical_type is None: @@ -85,6 +94,17 @@ def update_loader_rows_schema( f"Source loader returned conflicting types for column '{column_name}'" ) next_inferred_types[column_name] = logical_type + next_shapes: dict[str, DecimalShape] = dict(decimal_shapes) + shape: DecimalShape + for column_name, shape in batch_shapes.items(): + previous_shape: DecimalShape | None = decimal_shapes.get(column_name) + if previous_shape is not None and previous_shape.widened(shape) != previous_shape: + raise ExecutorInputError( + f"Source loader column '{column_name}' needs {shape.widened(previous_shape)}, " + f"wider than the {previous_shape} its earlier rows gave it; declare the " + "column's type in the source's columns" + ) + next_shapes.setdefault(column_name, shape) added_columns: tuple[ColumnInfo, ...] = tuple( ColumnInfo( name=column_name, @@ -94,6 +114,7 @@ def update_loader_rows_schema( column_name=column_name, declared_types=declared_types, inferred_types=next_inferred_types, + decimal_shapes=next_shapes, ) or "VARCHAR" ), @@ -104,6 +125,7 @@ def update_loader_rows_schema( column_names=tuple(names), inferred_types=next_inferred_types, added_columns=added_columns, + decimal_shapes=next_shapes, ) @@ -128,6 +150,7 @@ def build_rows_sql( columns: tuple[SourceColumnEntry, ...], column_names: tuple[str, ...] | None = None, inferred_types: dict[str, LoaderLogicalType] | None = None, + decimal_shapes: Mapping[str, DecimalShape] | None = None, ) -> str: """Build an adapter-rendered SELECT for framework-managed loader rows.""" @@ -136,8 +159,14 @@ def build_rows_sql( ) if not resolved_column_names: raise ExecutorInputError("Source loader returned no rows and source declares no columns") + declared_types: dict[str, str] = _declared_types(columns=columns) resolved_inferred_types: dict[str, LoaderLogicalType] = inferred_types or _infer_column_types( - rows=rows + rows=rows, declared_types=declared_types + ) + resolved_shapes: Mapping[str, DecimalShape] = ( + _decimal_shapes(rows=rows, declared_types=declared_types) + if decimal_shapes is None + else decimal_shapes ) return adapter.render_loader_rows_select( rows=rows, @@ -147,6 +176,7 @@ def build_rows_sql( columns=columns, column_names=resolved_column_names, inferred_types=resolved_inferred_types, + decimal_shapes=resolved_shapes, ), inferred_types=resolved_inferred_types, ) @@ -158,6 +188,7 @@ def loader_column_sql_types( columns: tuple[SourceColumnEntry, ...], column_names: tuple[str, ...], inferred_types: dict[str, LoaderLogicalType], + decimal_shapes: Mapping[str, DecimalShape], ) -> dict[str, str]: """Return each loader column's declared or inferred warehouse type, when known.""" @@ -171,6 +202,7 @@ def loader_column_sql_types( column_name=column_name, declared_types=declared_types, inferred_types=inferred_types, + decimal_shapes=decimal_shapes, ) ) is not None @@ -200,6 +232,7 @@ def _column_sql_type( column_name: str, declared_types: dict[str, str], inferred_types: dict[str, LoaderLogicalType], + decimal_shapes: Mapping[str, DecimalShape], ) -> str | None: declared_type: str | None = declared_types.get(column_name) if declared_type is not None: @@ -207,17 +240,67 @@ def _column_sql_type( inferred_type: LoaderLogicalType | None = inferred_types.get(column_name) if inferred_type is None: return None - return adapter.render_loader_logical_type(inferred_type) + shape: DecimalShape | None = decimal_shapes.get(column_name) + if shape is None: + return adapter.render_loader_logical_type(type_name=inferred_type) + return adapter.render_loader_logical_type( + type_name=inferred_type, precision=shape.precision, scale=shape.scale + ) -def _infer_column_types(rows: tuple[dict[str, object], ...]) -> dict[str, LoaderLogicalType]: +def _decimal_shapes( + *, rows: tuple[dict[str, object], ...], declared_types: Mapping[str, str] +) -> dict[str, DecimalShape]: + """The narrowest decimal type holding every Decimal of each undeclared column.""" + + shapes: dict[str, DecimalShape] = {} + row: dict[str, object] + for row in rows: + column_name: str + value: object + for column_name, value in row.items(): + if column_name in declared_types or not isinstance(value, Decimal): + continue + shape: DecimalShape = _decimal_shape(column_name=column_name, value=value) + previous: DecimalShape | None = shapes.get(column_name) + shapes[column_name] = shape if previous is None else previous.widened(shape) + return shapes + + +def _decimal_shape(*, column_name: str, value: Decimal) -> DecimalShape: + if not value.is_finite(): + raise ExecutorInputError( + f"Source loader returned the Decimal {value} for column '{column_name}', which no " + "DECIMAL column can hold" + ) + digit_count: int = len(value.as_tuple().digits) + exponent: int = int(value.as_tuple().exponent) + shape: DecimalShape = DecimalShape( + integer_digits=max(digit_count + exponent, 0), scale=max(-exponent, 0) + ) + if shape.precision > _MAX_DECIMAL_PRECISION: + raise ExecutorInputError( + f"Source loader returned the Decimal {value} for column '{column_name}', which needs " + f"more than {_MAX_DECIMAL_PRECISION} digits; declare the column's type in the " + "source's columns" + ) + return shape + + +def _infer_column_types( + *, rows: tuple[dict[str, object], ...], declared_types: Mapping[str, str] +) -> dict[str, LoaderLogicalType]: inferred: dict[str, LoaderLogicalType] = {} row: dict[str, object] for row in rows: column_name: str value: object for column_name, value in row.items(): - logical_type: LoaderLogicalType | None = _infer_value_type(value) + if column_name in declared_types: + continue + logical_type: LoaderLogicalType | None = _infer_value_type( + column_name=column_name, value=value + ) if logical_type is None: continue previous_type: LoaderLogicalType | None = inferred.get(column_name) @@ -229,15 +312,17 @@ def _infer_column_types(rows: tuple[dict[str, object], ...]) -> dict[str, Loader return inferred -def _infer_value_type(value: object) -> LoaderLogicalType | None: +def _infer_value_type(*, column_name: str, value: object) -> LoaderLogicalType | None: if value is None: return None if isinstance(value, bool): return LoaderLogicalType.BOOLEAN if isinstance(value, int): return LoaderLogicalType.INTEGER - if isinstance(value, float | Decimal): + if isinstance(value, float): return LoaderLogicalType.FLOAT + if isinstance(value, Decimal): + return LoaderLogicalType.DECIMAL if isinstance(value, str): return LoaderLogicalType.STRING if isinstance(value, datetime): @@ -246,4 +331,8 @@ def _infer_value_type(value: object) -> LoaderLogicalType | None: return LoaderLogicalType.DATE if isinstance(value, dict | list): return LoaderLogicalType.JSON - return LoaderLogicalType.STRING + raise ExecutorInputError( + f"Source loader returned a {type(value).__name__} value for column '{column_name}', " + "which has no declared type; declare the column's type in the source's columns, or " + "return a str, int, float, Decimal, bool, date, datetime, dict or list" + ) diff --git a/src/sqlbuild/executor/load/_helpers/staging.py b/src/sqlbuild/executor/load/_helpers/staging.py index 759a5af123..0291b18d29 100644 --- a/src/sqlbuild/executor/load/_helpers/staging.py +++ b/src/sqlbuild/executor/load/_helpers/staging.py @@ -17,7 +17,7 @@ loader_column_sql_types, update_loader_rows_schema, ) -from sqlbuild.executor.load.models import LoaderRowsSchema +from sqlbuild.executor.load.models import DecimalShape, LoaderRowsSchema from sqlbuild.executor.run.constants import STAGING_RELATION_SUFFIX @@ -38,6 +38,7 @@ def write_loader_rows_to_staging( staging_created: bool = False column_names: tuple[str, ...] = tuple(column.name for column in source_entry.columns) inferred_types: dict[str, LoaderLogicalType] = {} + decimal_shapes: dict[str, DecimalShape] = {} adapter.drop( connection=connection, destination=staging, @@ -52,10 +53,12 @@ def write_loader_rows_to_staging( columns=source_entry.columns, column_names=column_names, inferred_types=inferred_types, + decimal_shapes=decimal_shapes, contract_enforced=source_entry.contract == ContractPolicy.ENFORCED, ) column_names = schema.column_names inferred_types = schema.inferred_types + decimal_shapes = schema.decimal_shapes if staging_created and schema.added_columns: adapter.add_columns( connection=connection, @@ -69,6 +72,7 @@ def write_loader_rows_to_staging( columns=source_entry.columns, column_names=column_names, inferred_types=inferred_types, + decimal_shapes=decimal_shapes, ) if staging_created: staging_adapter: BaseAdapter = adapter @@ -89,6 +93,7 @@ def write_loader_rows_to_staging( source_entry=source_entry, column_names=column_names, inferred_types=inferred_types, + decimal_shapes=decimal_shapes, ), statement_recorder=statement_recorder, ) @@ -101,6 +106,7 @@ def write_loader_rows_to_staging( columns=source_entry.columns, column_names=column_names, inferred_types=inferred_types, + decimal_shapes=decimal_shapes, ) adapter.create_table_as( connection=connection, @@ -111,6 +117,7 @@ def write_loader_rows_to_staging( source_entry=source_entry, column_names=column_names, inferred_types=inferred_types, + decimal_shapes=decimal_shapes, ), statement_recorder=statement_recorder, ) @@ -123,6 +130,7 @@ def _staging_config( source_entry: SourceEntry, column_names: tuple[str, ...], inferred_types: dict[str, LoaderLogicalType], + decimal_shapes: dict[str, DecimalShape], ) -> dict[str, Any]: """Hand loader column types to adapters whose CTAS cannot keep them.""" @@ -132,6 +140,7 @@ def _staging_config( columns=source_entry.columns, column_names=column_names, inferred_types=inferred_types, + decimal_shapes=decimal_shapes, ) } diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 39097300b7..9d5d7d8e85 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -31,6 +31,31 @@ from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, ExecutionResourceKind +@dataclass(frozen=True) +class DecimalShape: + """The digits a decimal column keeps before and after its point.""" + + integer_digits: int + scale: int + + @property + def precision(self) -> int: + """The total digits, at least one.""" + + return max(self.integer_digits + self.scale, 1) + + def widened(self, other: DecimalShape) -> DecimalShape: + """The narrowest shape holding the values of both.""" + + return DecimalShape( + integer_digits=max(self.integer_digits, other.integer_digits), + scale=max(self.scale, other.scale), + ) + + def __str__(self) -> str: + return f"DECIMAL({self.precision}, {self.scale})" + + @dataclass(frozen=True) class LoaderRowsSchema: """Tracked schema state for one or more loader row batches.""" @@ -38,6 +63,7 @@ class LoaderRowsSchema: column_names: tuple[str, ...] inferred_types: dict[str, LoaderLogicalType] added_columns: tuple[ColumnInfo, ...] + decimal_shapes: dict[str, DecimalShape] @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 09a3dc543a..3a73f620b9 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CursorBounds, + IncrementalMode, IncrementalStrategy, ModelPlanEntry, OnSchemaChange, @@ -875,6 +876,16 @@ def _apply_schema_change( return () +def _cursor_replacement_key( + *, entry: ModelPlanEntry, unique_key: tuple[str, ...] +) -> tuple[str, ...]: + """The key whose rows a cursor delete_insert also replaces; a microbatch keeps its window.""" + + if entry.incremental_mode == IncrementalMode.MICROBATCH: + return () + return unique_key + + def _execute_dml( *, adapter: BaseAdapter, @@ -926,6 +937,7 @@ def _execute_dml( columns=dml_columns, statement_recorder=statement_recorder, cursor_type=entry.cursor_type, + unique_key=_cursor_replacement_key(entry=entry, unique_key=unique_key), ) return adapter.delete_insert( connection=connection, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py index 5189c5c77e..6f69c126b0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_janitor_usage.py @@ -92,27 +92,27 @@ def test_given_query_log_when_running_janitor_then_view_chain_reads_are_in_use( "test_case", ( ClickHouseJanitorUsageUncheckedE2ETestCase( - description="a janitor role that may not read system.query_log is not checked", + description="--keep-in-use stops when the role may not read system.query_log", connection_settings="", grants=(), expected_fragments=( - "Relation usage not checked: cannot read system.query_log (", - " usage not checked: cannot read system.query_log (needs query logging on " - "and SELECT on system.query_log)\n", + "usage was not checked: cannot read system.query_log (needs query logging on " + "and SELECT on system.query_log); nothing was archived", ), ), ClickHouseJanitorUsageUncheckedE2ETestCase( - description="query logging turned off for the connection is not checked", + description="--keep-in-use stops when query logging is off for the connection", connection_settings="\n[connection.settings]\nlog_queries = 0\n", grants=("GRANT SELECT ON system.query_log TO {user}",), expected_fragments=( - " usage not checked: query logging is off for this connection (log_queries = 0)\n", + "usage was not checked: query logging is off for this connection " + "(log_queries = 0); nothing was archived", ), ), ), ids=lambda case: case.description, ) -def test_given_no_query_log_when_running_janitor_then_usage_is_not_checked( +def test_given_no_query_log_when_keeping_in_use_then_janitor_stops_and_archives_nothing( test_case: ClickHouseJanitorUsageUncheckedE2ETestCase, clickhouse_e2e_config: dict[str, object], janitor_schema: str, @@ -135,14 +135,18 @@ def test_given_no_query_log_when_running_janitor_then_usage_is_not_checked( schema=janitor_schema, connection_settings=test_case.connection_settings, ) + stale: tuple[str, ...] = stale_relation_names( + config=clickhouse_e2e_config, schema=janitor_schema + ) result: subprocess.CompletedProcess[str] = run_sqb( command=(*APPROVED_JANITOR, "--keep-in-use"), project_dir=project_dir ) - assert result.returncode == 0, result.stdout + result.stderr + assert result.returncode != 0, result.stdout + output: str = " ".join((result.stdout + result.stderr).split()) fragment: str for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - assert "nothing kept in use: no usage could be checked" in result.stdout - assert stale_relation_names(config=clickhouse_e2e_config, schema=janitor_schema) == () + assert fragment in output, output + assert stale + assert stale_relation_names(config=clickhouse_e2e_config, schema=janitor_schema) == stale diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py index d323f908b4..d65d2d5f71 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py @@ -68,6 +68,45 @@ def test_given_debug_command_when_running_then_outputs_checks( assert test_case.expected_stdout_fragment in result.stdout +@pytest.mark.parametrize( + "test_case", + [ + DebugCliTestCase( + description="reports the defaults location models use, not the connection's", + command=("debug", "--no-connection"), + expected_stdout_fragment=( + " database: DEFAULTS_DB [OK resolved]\n schema: DEFAULTS_SCHEMA [OK resolved]\n" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_defaults_location_when_debugging_then_reports_it( + tmp_path: Path, + test_case: DebugCliTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="debug_defaults", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "debug_defaults"\nadapter = "snowflake"\n\n' + '[connection]\naccount = "acme"\nuser = "u"\npassword = "p"\n' + 'database = "CONN_DB"\nschema = "CONN_SCHEMA"\n\n' + '[defaults]\ndatabase = "DEFAULTS_DB"\nschema = "DEFAULTS_SCHEMA"\n' + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_returncode, result.stderr + assert test_case.expected_stdout_fragment in result.stdout + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index ed9c45ac8c..eb7f03df0a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -339,6 +339,17 @@ class JanitorUsageJsonE2ETestCase: expected_archived: int +@dataclass(frozen=True) +class JanitorKeepInUseIncompleteE2ETestCase: + """Test case for `--keep-in-use` when usage is unavailable or only partly read.""" + + description: str + reads: dict[str, tuple[tuple[str, ...], int]] + unavailable: str | None + notices: tuple[str, ...] + expected_error: str + + @dataclass(frozen=True) class JanitorUsageFailureE2ETestCase: """Test case for janitor runs whose usage source fails outright.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index b6ca3a8f9d..bba3549902 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -847,6 +847,7 @@ def relation_usage( for relation in relations if relation.name in reads ), + notices=tuple(settings["notices"]), ) """ @@ -907,10 +908,12 @@ def write_usage_settings( unavailable: str | None = None, via: Mapping[str, tuple[str, ...]] = MappingProxyType({}), failure: str | None = None, + notices: tuple[str, ...] = (), ) -> None: """Run janitor on the usage adapter: who read which relation, or why usage is unavailable. - A relation in `via` was read only through the named objects; `failure` makes the read fail. + A relation in `via` was read only through the named objects; `failure` makes the read fail; + `notices` say what a completed check left out. """ (project_dir / "sqlbuild_local.toml").write_text( @@ -931,6 +934,7 @@ def write_usage_settings( }, "unavailable": unavailable, "failure": failure, + "notices": list(notices), } ), encoding="utf-8", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py index 2d8d724049..d0a1b31489 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor_usage.py @@ -10,6 +10,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( + JanitorKeepInUseIncompleteE2ETestCase, JanitorUsageE2ETestCase, JanitorUsageFailureE2ETestCase, JanitorUsageJsonE2ETestCase, @@ -54,17 +55,6 @@ ), expected_archived=3, ), - JanitorUsageUncheckedE2ETestCase( - description="--keep-in-use without a usage source warns and archives everything", - janitor_command=(*APPROVED_JANITOR, "--keep-in-use"), - expected_ordered_lines=( - " usage not checked: unsupported adapter\n" - " nothing kept in use: no usage could be checked, so relations in use are " - "archived too\n", - "\u2713 Janitor completed archived 3, deleted 1, pruned 2", - ), - expected_archived=3, - ), ], ids=lambda case: case.description, ) @@ -404,5 +394,57 @@ def test_given_failing_usage_source_when_running_janitor_then_failure_line_is_pr assert len(list_archive_names(db_path=project_dir / "janitor.duckdb")) == 1 +@pytest.mark.parametrize( + "test_case", + [ + JanitorKeepInUseIncompleteE2ETestCase( + description="--keep-in-use stops when usage cannot be read", + reads={}, + unavailable="cannot read access history", + notices=(), + expected_error=( + "keep_in_use needs the usage of every relation to archive, but usage was not " + "checked: cannot read access history; nothing was archived" + ), + ), + JanitorKeepInUseIncompleteE2ETestCase( + description="--keep-in-use stops when usage is only partly read", + reads={"old_orders_000": (("alice",), 3)}, + unavailable=None, + notices=("view chains not followed: grant the dependency role",), + expected_error=( + "keep_in_use needs the usage of every relation to archive, but usage was only " + "partly checked: view chains not followed: grant the dependency role; nothing " + "was archived" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_incomplete_usage_when_keeping_in_use_then_janitor_stops_and_archives_nothing( + test_case: JanitorKeepInUseIncompleteE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_usage_janitor_project( + tmp_path=tmp_path, project_name="janitor_keep_incomplete", stale_tables=2 + ) + write_usage_settings( + project_dir=project_dir, + reads=test_case.reads, + unavailable=test_case.unavailable, + notices=test_case.notices, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=(*APPROVED_JANITOR, "--keep-in-use"), project_dir=project_dir + ) + + assert result.returncode != 0, result.stdout + output: str = " ".join((result.stdout + result.stderr).split()) + assert test_case.expected_error in output, output + assert "Relation usage check incomplete. (" in result.stdout + result.stderr + assert "Janitor completed" not in result.stdout + assert len(list_archive_names(db_path=project_dir / "janitor.duckdb")) == 1 + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py index 93802fa307..1c57b6bb82 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/helpers.py @@ -322,6 +322,17 @@ def _cursor_oracle(test_case: ClickHouseRewriteTestCase) -> tuple[str, ...]: ) +def _keyed_cursor_oracle(test_case: ClickHouseRewriteTestCase) -> tuple[str, ...]: + assert test_case.window is not None + keys: str = ", ".join(test_case.unique_key) + return ( + f"DELETE FROM orders WHERE (order_date >= '{test_case.window[0]}' " + f"AND order_date < '{test_case.window[1]}') " + f"OR ({keys}) IN (SELECT {keys} FROM incoming)", + "INSERT INTO orders SELECT * FROM incoming", + ) + + def _merge_oracle(test_case: ClickHouseRewriteTestCase) -> tuple[str, ...]: updated_columns: list[str] = sorted( set(ORDER_COLUMN_NAMES[1:]) - set(test_case.exclude_columns) @@ -392,6 +403,29 @@ def _write_cursor( ) +def _write_keyed_cursor( + *, + adapter: ClickHouseAdapter, + connection: Any, + test_case: ClickHouseRewriteTestCase, + relation: str, + sql: str, +) -> None: + assert test_case.window is not None + adapter.delete_insert_cursor( + connection=connection, + destination=relation, + sql=sql, + cursor_column="order_date", + cursor_start=test_case.window[0], + cursor_end=test_case.window[1], + columns=ORDER_COLUMN_NAMES, + statement_recorder=StatementRecorder(), + cursor_type="timestamp", + unique_key=test_case.unique_key, + ) + + def _write_merge( *, adapter: ClickHouseAdapter, @@ -414,12 +448,14 @@ def _write_merge( "append": _append_oracle, "delete_insert": _delete_insert_oracle, "cursor": _cursor_oracle, + "keyed_cursor": _keyed_cursor_oracle, "merge": _merge_oracle, } _WRITERS: dict[str, Callable[..., None]] = { "append": _write_append, "delete_insert": _write_delete_insert, "cursor": _write_cursor, + "keyed_cursor": _write_keyed_cursor, "merge": _write_merge, } diff --git a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py index 5e4337fc3b..5e770d89d2 100644 --- a/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/clickhouse/test_client.py @@ -429,6 +429,14 @@ def test_given_framework_state_ddl_when_creating_then_clickhouse_accepts_every_t incoming_rows=((9, "iv", 90, "2026-01-21"),), window=("2026-01-15", "2026-02-10"), ), + ClickHouseRewriteTestCase( + description="keyed cursor window also replaces a key from another partition (partitioned)", + strategy="keyed_cursor", + partition_by=_PARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((1, "ada", 11, "2026-01-21"), (9, "iv", 90, "2026-01-22")), + window=("2026-01-15", "2026-02-10"), + ), ClickHouseRewriteTestCase( description="empty window clears its rows (partitioned)", strategy="cursor", @@ -471,6 +479,14 @@ def test_given_framework_state_ddl_when_creating_then_clickhouse_accepts_every_t incoming_rows=((9, "iv", 90, "2026-01-21"),), window=("2026-01-15", "2026-02-10"), ), + ClickHouseRewriteTestCase( + description="keyed cursor window also replaces a key from another partition (unpartitioned)", + strategy="keyed_cursor", + partition_by=_UNPARTITIONED, + initial_rows=INITIAL_ORDERS, + incoming_rows=((1, "ada", 11, "2026-01-21"), (9, "iv", 90, "2026-01-22")), + window=("2026-01-15", "2026-02-10"), + ), ClickHouseRewriteTestCase( description="empty window clears its rows (unpartitioned)", strategy="cursor", diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py index fcf5cad5f0..0584f027d5 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py @@ -202,6 +202,20 @@ def _cursor_window(adapter: SqliteAdapter, connection: Any, sql: str) -> None: ) +def _keyed_cursor_window(adapter: SqliteAdapter, connection: Any, sql: str) -> None: + adapter.delete_insert_cursor( + connection=connection, + destination="orders", + sql=sql, + cursor_column="ordered_at", + cursor_start="2026-01-02 00:00:00", + cursor_end="2026-01-03 00:00:00", + cursor_type="timestamp", + statement_recorder=recorder(), + unique_key=("order_id",), + ) + + def _add_column(adapter: SqliteAdapter, connection: Any) -> None: adapter.add_columns( connection=connection, @@ -245,6 +259,7 @@ def _retype_column(adapter: SqliteAdapter, connection: Any) -> None: "merge": _merge, "delete_insert": _delete_insert, "cursor": _cursor_window, + "keyed_cursor": _keyed_cursor_window, } ) _EVOLUTIONS: MappingProxyType[str, Callable[[SqliteAdapter, Any], None]] = MappingProxyType( diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py index 5afc6b4e14..b9310a1a5a 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py @@ -383,6 +383,16 @@ def test_given_contract_type_when_casting_then_sqlite_renders_storage_form( ), expected_rows=((1, "placed"), (3, "placed"), (9, "placed")), ), + SqliteStrategyTestCase( + description="keyed cursor delete_insert also replaces a key whose cursor moved", + strategy="keyed_cursor", + select_columns="order_id, status", + source_sql=( + "SELECT 1 AS order_id, 10 AS customer_id, 13 AS amount, " + "'2026-01-02 15:00:00' AS ordered_at, 'shipped' AS status" + ), + expected_rows=((1, "shipped"), (3, "placed")), + ), ], ids=lambda case: case.description, ) @@ -421,6 +431,11 @@ def test_given_strategy_when_writing_incrementally_then_sqlite_produces_expected strategy="cursor", expected_rows=((1,), (2,), (3,)), ), + SqliteAtomicityTestCase( + description="a failing keyed cursor window leaves the target unchanged", + strategy="keyed_cursor", + expected_rows=((1,), (2,), (3,)), + ), SqliteAtomicityTestCase( description="a failing merge leaves the target unchanged", strategy="merge", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 5e172c1c6b..6aa83438cf 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -4,6 +4,7 @@ from collections import defaultdict from collections.abc import Callable from datetime import date, datetime +from decimal import Decimal from pathlib import Path from typing import Any, cast @@ -3924,6 +3925,70 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri assert column_types[expected_column] == expected_type +@pytest.mark.parametrize( + "test_case", + [ + LoadCommandInferredColumnsTestCase( + description="decimals keep their precision and declared columns take any value", + project_files={ + "sqlbuild_project.toml": _PROJECT_FILE, + "sources/raw.yml": """ +sources: + - name: raw_inferred + description: Test source raw_inferred. + managed: true + write_strategy: table + columns: + - name: order_ref + type: VARCHAR +""".strip() + + "\n", + "python/loaders/raw.py": """ +from decimal import Decimal +from uuid import UUID + +from sqlbuild.loaders import loader + +@loader +def raw_inferred(ctx): + '''Test loader raw_inferred.''' + yield {"order_ref": UUID("12345678-1234-5678-1234-567812345678"), "price": Decimal("12.50")} + yield {"order_ref": None, "price": Decimal("1234.5")} +""", + }, + expected_row=("12345678-1234-5678-1234-567812345678", Decimal("12.50")), + expected_column_types={"order_ref": "VARCHAR", "price": "DECIMAL(6,2)"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_decimal_and_declared_values_when_running_load_then_keeps_their_types( + test_case: LoadCommandInferredColumnsTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files(tmp_path, test_case.project_files) + + exit_code: int = run_load_command( + request=LoadCommandRequest(project_dir=tmp_path, no_color=True) + ) + + assert exit_code == 0 + connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) + try: + row: tuple[object, ...] | None = connection.execute( + "SELECT order_ref, price FROM raw_inferred ORDER BY price LIMIT 1" + ).fetchone() + column_rows: list[tuple[str, str]] = connection.execute( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_name = 'raw_inferred'" + ).fetchall() + finally: + connection.close() + assert row == test_case.expected_row + assert dict(column_rows) == test_case.expected_column_types + + @pytest.mark.parametrize( "test_case", [ @@ -4514,6 +4579,66 @@ def raw_conflict(ctx): expected_exit_code=1, expected_stdout_fragment="conflicting types for column 'id'", ), + LoadCommandFailureTestCase( + description="fails clearly when an undeclared column holds an unrecognised type", + project_files={ + "sqlbuild_project.toml": _PROJECT_FILE, + "sources/raw.yml": """ +sources: + - name: raw_unknown_type + description: Test source raw_unknown_type. + managed: true + write_strategy: table +""".strip() + + "\n", + "python/loaders/raw.py": """ +from uuid import UUID + +from sqlbuild.loaders import loader + +@loader +def raw_unknown_type(ctx): + '''Test loader raw_unknown_type.''' + return [{"id": 1, "order_ref": UUID("12345678-1234-5678-1234-567812345678")}] +""", + }, + expected_exit_code=1, + expected_stdout_fragment=( + "Source loader returned a UUID value for column 'order_ref', which has no " + "declared type" + ), + ), + LoadCommandFailureTestCase( + description="fails clearly when a later batch needs a wider decimal", + project_files={ + "sqlbuild_project.toml": _PROJECT_FILE, + "sources/raw.yml": """ +sources: + - name: raw_wider_decimal + description: Test source raw_wider_decimal. + managed: true + write_strategy: table + load_batch_size: 1 +""".strip() + + "\n", + "python/loaders/raw.py": """ +from decimal import Decimal + +from sqlbuild.loaders import loader + +@loader +def raw_wider_decimal(ctx): + '''Test loader raw_wider_decimal.''' + yield {"price": Decimal("1.5")} + yield {"price": Decimal("123.25")} +""", + }, + expected_exit_code=1, + expected_stdout_fragment=( + "Source loader column 'price' needs DECIMAL(5, 2), wider than the " + "DECIMAL(2, 1) its earlier rows gave it" + ), + ), LoadCommandFailureTestCase( description="fails when loader returns rows without write strategy", project_files={ diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index d0feea6825..727a3a2a26 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -600,7 +600,24 @@ def test_given_snowflake_local_override_without_target_namespace_when_compiling_ }, expected_database="LOCAL_DB", expected_schema="LOCAL_SCHEMA", - ) + ), + SnowflakeTargetValidationIntegrationTestCase( + description="snowflake defaults namespace resolves the project target like models", + project_files={ + "sqlbuild_project.toml": ( + 'name = "demo"\nadapter = "snowflake"\n\n' + "[connection]\n" + 'database = "CONN_DB"\n' + 'schema = "CONN_SCHEMA"\n\n' + "[defaults]\n" + 'database = "DEFAULTS_DB"\n' + 'schema = "DEFAULTS_SCHEMA"\n' + ), + "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n", + }, + expected_database="DEFAULTS_DB", + expected_schema="DEFAULTS_SCHEMA", + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py index e3829b48c4..a7789e5e24 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py @@ -99,6 +99,47 @@ @pytest.mark.parametrize( "test_case", [ + BuildExecutionTestCase( + description="cursor delete_insert with a unique_key replaces a row whose cursor moved", + project_files={ + "sqlbuild_project.toml": _PROJECT_YML, + "models/orders.sql": ( + "MODEL (description 'Latest version of each order.',\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key [id],\n" + " cursor updated_at,\n" + " cursor_type timestamp,\n" + " cursor_grain second,\n" + ");\n\n" + 'SELECT id, updated_at FROM __source("raw_orders")' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + ), + }, + setup_sql=( + "CREATE TABLE main.orders AS SELECT * FROM (VALUES " + "(1, TIMESTAMP '2024-01-01'), (2, TIMESTAMP '2024-01-02')) AS t(id, updated_at)", + "CREATE TABLE raw_orders AS SELECT * FROM (VALUES " + "(1, TIMESTAMP '2024-01-05'), (2, TIMESTAMP '2024-01-02'), " + "(3, TIMESTAMP '2024-01-05')) AS t(id, updated_at)", + ), + expected_status=BuildStatus.SUCCESS, + expected_success_count=1, + expected_model_statuses=(("orders", ExecutionStatus.SUCCESS),), + expected_query_results=( + ( + "SELECT id, CAST(updated_at AS VARCHAR) FROM main.orders ORDER BY id", + ( + (1, "2024-01-05 00:00:00"), + (2, "2024-01-02 00:00:00"), + (3, "2024-01-05 00:00:00"), + ), + ), + ), + ), BuildExecutionTestCase( description="sql udf builds before dependent model", project_files={ diff --git a/tests/unit/src/sqlbuild/adapter/contract/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/_test_types.py index 4f6c12ef80..bd529fc9f7 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/_test_types.py @@ -12,6 +12,15 @@ class AdapterUserErrorTestCase: expected_args: tuple[str] +@dataclass(frozen=True) +class AdapterLoaderDecimalTypeTestCase: + description: str + adapter_name: str + precision: int + scale: int + expected_type: str + + @dataclass(frozen=True) class AdapterLoaderTypeMappingTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapter/contract/test_loader_types.py b/tests/unit/src/sqlbuild/adapter/contract/test_loader_types.py index 435ef9fcd2..433f076f8a 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/test_loader_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/test_loader_types.py @@ -9,6 +9,7 @@ from sqlbuild.adapter.discovery.main.builtins import builtin_adapter_classes from tests.unit.src.sqlbuild.adapter.contract._test_types import ( AdapterIdentifierRenderingTestCase, + AdapterLoaderDecimalTypeTestCase, AdapterLoaderJsonRowsTestCase, AdapterLoaderRowsEmptySelectTestCase, AdapterLoaderRowsSelectTestCase, @@ -21,6 +22,7 @@ LoaderLogicalType.BOOLEAN: "BOOLEAN", LoaderLogicalType.INTEGER: "BIGINT", LoaderLogicalType.FLOAT: "DOUBLE", + LoaderLogicalType.DECIMAL: "DECIMAL(38, 0)", LoaderLogicalType.STRING: "VARCHAR", LoaderLogicalType.TIMESTAMP: "TIMESTAMP", LoaderLogicalType.DATE: "DATE", @@ -49,13 +51,18 @@ LoaderLogicalType.BOOLEAN: "BOOL", LoaderLogicalType.INTEGER: "INT64", LoaderLogicalType.FLOAT: "FLOAT64", + LoaderLogicalType.DECIMAL: "BIGNUMERIC(38, 0)", LoaderLogicalType.STRING: "STRING", }, ), AdapterLoaderTypeMappingTestCase( description="snowflake maps json to variant", adapter_name=BuiltinAdapter.SNOWFLAKE.value, - expected_types={**_DEFAULT_EXPECTED_TYPES, LoaderLogicalType.JSON: "VARIANT"}, + expected_types={ + **_DEFAULT_EXPECTED_TYPES, + LoaderLogicalType.DECIMAL: "NUMBER(38, 0)", + LoaderLogicalType.JSON: "VARIANT", + }, ), AdapterLoaderTypeMappingTestCase( description="databricks maps strings and json to string", @@ -72,6 +79,7 @@ expected_types={ **_DEFAULT_EXPECTED_TYPES, LoaderLogicalType.FLOAT: "DOUBLE PRECISION", + LoaderLogicalType.DECIMAL: "NUMERIC(38, 0)", LoaderLogicalType.STRING: "TEXT", LoaderLogicalType.JSON: "JSONB", }, @@ -88,11 +96,60 @@ def test_given_builtin_adapter_when_rendering_loader_types_then_maps_every_logic logical_type: LoaderLogicalType for logical_type in LoaderLogicalType: assert ( - adapter.render_loader_logical_type(logical_type) + adapter.render_loader_logical_type(type_name=logical_type) == test_case.expected_types[logical_type] ) +@pytest.mark.parametrize( + "test_case", + [ + AdapterLoaderDecimalTypeTestCase( + description="bigquery keeps a decimal that fits NUMERIC in NUMERIC", + adapter_name=BuiltinAdapter.BIGQUERY.value, + precision=38, + scale=9, + expected_type="NUMERIC(38, 9)", + ), + AdapterLoaderDecimalTypeTestCase( + description="bigquery moves a decimal with ten fractional digits to BIGNUMERIC", + adapter_name=BuiltinAdapter.BIGQUERY.value, + precision=12, + scale=10, + expected_type="BIGNUMERIC(12, 10)", + ), + AdapterLoaderDecimalTypeTestCase( + description="bigquery moves a decimal with thirty integer digits to BIGNUMERIC", + adapter_name=BuiltinAdapter.BIGQUERY.value, + precision=32, + scale=2, + expected_type="BIGNUMERIC(32, 2)", + ), + AdapterLoaderDecimalTypeTestCase( + description="clickhouse wraps a decimal in Nullable", + adapter_name=BuiltinAdapter.CLICKHOUSE.value, + precision=6, + scale=2, + expected_type="Nullable(Decimal(6, 2))", + ), + ], + ids=lambda case: case.description, +) +def test_given_decimal_shape_when_rendering_loader_type_then_uses_its_precision_and_scale( + test_case: AdapterLoaderDecimalTypeTestCase, +) -> None: + adapter: BaseAdapter = builtin_adapter_classes()[test_case.adapter_name]() + + assert ( + adapter.render_loader_logical_type( + type_name=LoaderLogicalType.DECIMAL, + precision=test_case.precision, + scale=test_case.scale, + ) + == test_case.expected_type + ) + + @pytest.mark.parametrize( "test_case", [ @@ -265,8 +322,8 @@ def test_given_builtin_adapter_when_rendering_loader_rows_then_returns_adapter_s rows=({"id": 1, "status": "placed"}, {"id": 2, "status": "shipped"}), column_names=("id", "status"), column_sql_types={ - "id": adapter.render_loader_logical_type(LoaderLogicalType.INTEGER), - "status": adapter.render_loader_logical_type(LoaderLogicalType.STRING), + "id": adapter.render_loader_logical_type(type_name=LoaderLogicalType.INTEGER), + "status": adapter.render_loader_logical_type(type_name=LoaderLogicalType.STRING), }, inferred_types={"id": LoaderLogicalType.INTEGER, "status": LoaderLogicalType.STRING}, ) @@ -312,8 +369,8 @@ def test_given_json_rows_when_rendering_loader_rows_then_values_hold_only_consta rows=test_case.rows, column_names=("id", "payload"), column_sql_types={ - "id": adapter.render_loader_logical_type(LoaderLogicalType.INTEGER), - "payload": adapter.render_loader_logical_type(LoaderLogicalType.JSON), + "id": adapter.render_loader_logical_type(type_name=LoaderLogicalType.INTEGER), + "payload": adapter.render_loader_logical_type(type_name=LoaderLogicalType.JSON), }, inferred_types={"id": LoaderLogicalType.INTEGER, "payload": LoaderLogicalType.JSON}, ) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index 3f61d0ccf0..29ef30218c 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -125,10 +125,11 @@ class DatabricksTableFreshnessBatchTestCase: @dataclass(frozen=True) -class DatabricksTableFreshnessFallbackTestCase: +class DatabricksTableFreshnessMixedBatchTestCase: description: str - expected_data_versions: tuple[datetime, ...] - expected_query_fragments: tuple[str, ...] + delta_version: datetime + view_error: str + expected_view_message: str @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index 122d6bce96..dc39e824a0 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -11,6 +11,7 @@ RelationInfo, TableFreshnessMetadata, TableFreshnessRequest, + TableFreshnessStatus, ) from sqlbuild.adapter.contract.types import FunctionNullabilityRule, LoaderLogicalType from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter @@ -30,7 +31,7 @@ DatabricksSchemaExistsTestCase, DatabricksStringTypeCastRenderingTestCase, DatabricksTableFreshnessBatchTestCase, - DatabricksTableFreshnessFallbackTestCase, + DatabricksTableFreshnessMixedBatchTestCase, ) from tests.unit.src.sqlbuild.adapters.databricks.helpers import ( FailingDatabricksMetadataCursor, @@ -239,56 +240,56 @@ def test_given_physical_tables_when_getting_freshness_metadata_then_databricks_u @pytest.mark.parametrize( "test_case", [ - DatabricksTableFreshnessFallbackTestCase( - description="falls back to unity catalog last altered", - expected_data_versions=( - datetime(2026, 1, 4, 3, 4, 5), - datetime(2026, 1, 5, 4, 5, 6), - ), - expected_query_fragments=( - "FROM `system`.`information_schema`.`tables`", - "last_altered", - "orders", + DatabricksTableFreshnessMixedBatchTestCase( + description="a view in the batch leaves only the view unknown", + delta_version=datetime(2026, 1, 4, 3, 4, 5), + view_error="DESCRIBE HISTORY is only supported for Delta tables\nstack line", + expected_view_message=( + "Databricks Delta history unavailable for orders_v: " + "DESCRIBE HISTORY is only supported for Delta tables" ), ) ], ids=lambda case: case.description, ) -def test_given_delta_history_unavailable_when_getting_metadata_then_databricks_uses_uc_last_altered( - test_case: DatabricksTableFreshnessFallbackTestCase, +def test_given_view_in_freshness_batch_when_getting_metadata_then_only_the_view_is_unknown( + test_case: DatabricksTableFreshnessMixedBatchTestCase, ) -> None: adapter: DatabricksAdapter = DatabricksAdapter() - requests: tuple[TableFreshnessRequest, ...] = ( - TableFreshnessRequest(database="hive_metastore", schema="raw", name="orders"), - TableFreshnessRequest(database="hive_metastore", schema="raw", name="customers"), + orders: TableFreshnessRequest = TableFreshnessRequest( + database="main", schema="raw", name="orders" ) - history_cursor: FakeDatabricksMetadataCursor = FailingDatabricksMetadataCursor( - execute_error=RuntimeError("system catalog unavailable") + orders_v: TableFreshnessRequest = TableFreshnessRequest( + database="main", schema="raw", name="orders_v" ) - uc_cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor( - rows=[ - ("hive_metastore", "raw", "orders", "MANAGED", test_case.expected_data_versions[0]), - ("hive_metastore", "raw", "customers", "EXTERNAL", test_case.expected_data_versions[1]), - ] + batch_cursor: FakeDatabricksMetadataCursor = FailingDatabricksMetadataCursor( + execute_error=RuntimeError(test_case.view_error) + ) + orders_cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor( + rows=[("main", "raw", "orders", test_case.delta_version)] + ) + view_cursor: FakeDatabricksMetadataCursor = FailingDatabricksMetadataCursor( + execute_error=RuntimeError(test_case.view_error) ) connection: FakeDatabricksMetadataConnection = FakeDatabricksMetadataConnection( - (history_cursor, uc_cursor) + (batch_cursor, orders_cursor, view_cursor) ) metadata_by_request: dict[TableFreshnessRequest, TableFreshnessMetadata] = ( - adapter.get_tables_freshness_metadata(connection=connection, requests=requests) + adapter.get_tables_freshness_metadata(connection=connection, requests=(orders, orders_v)) ) - assert ( - tuple(metadata_by_request[request].data_version for request in requests) - == test_case.expected_data_versions + assert metadata_by_request[orders].status == TableFreshnessStatus.OBSERVED + assert metadata_by_request[orders].data_version == test_case.delta_version + assert metadata_by_request[orders_v].status == TableFreshnessStatus.UNAVAILABLE + assert metadata_by_request[orders_v].message == test_case.expected_view_message + assert orders_cursor.executed_sql is not None + assert "DESCRIBE HISTORY `main`.`raw`.`orders`" in orders_cursor.executed_sql + assert "UNION ALL" not in orders_cursor.executed_sql + assert all(cursor.closed for cursor in connection.returned_cursors) + assert "information_schema" not in "".join( + cursor.executed_sql or "" for cursor in connection.returned_cursors ) - assert all(metadata.value_kind == "timestamp" for metadata in metadata_by_request.values()) - assert uc_cursor.executed_sql is not None - for fragment in test_case.expected_query_fragments: - assert fragment in uc_cursor.executed_sql - assert history_cursor.closed is True - assert uc_cursor.closed is True @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index b17b2d3e6e..5a0d94f556 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -782,6 +782,40 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph expected_effective_target_database="target_db", expected_effective_target_schema="target_schema", ), + AssembleCompiledProjectEffectiveTargetTestCase( + description="prefers defaults location over connection fallback", + project_toml=""" +name = "demo" +adapter = "postgres" +default_target = "dev" + +[defaults] +database = "defaults_db" +schema = "defaults_schema" + +[targets.dev.connection] +database = "connection_db" +schema = "connection_schema" +""".strip() + + "\n", + expected_effective_target_database="defaults_db", + expected_effective_target_schema="defaults_schema", + ), + AssembleCompiledProjectEffectiveTargetTestCase( + description="never falls back to the connection location on snowflake", + project_toml=""" +name = "demo" +adapter = "snowflake" +default_target = "dev" + +[targets.dev.connection] +database = "connection_db" +schema = "connection_schema" +""".strip() + + "\n", + expected_effective_target_database=None, + expected_effective_target_schema=None, + ), ), ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 36ac4605f2..2ea022a89c 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -316,6 +316,7 @@ def delete_insert_cursor( columns: tuple[str, ...] | None = None, statement_recorder: StatementRecorder, cursor_type: str | None = None, + unique_key: tuple[str, ...] = (), ) -> None: raise NotImplementedError diff --git a/tests/unit/src/sqlbuild/executor/load/test_rows.py b/tests/unit/src/sqlbuild/executor/load/test_rows.py index b4b006dfaf..a23a7c0eca 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_rows.py +++ b/tests/unit/src/sqlbuild/executor/load/test_rows.py @@ -54,7 +54,7 @@ 'CAST("extra_bool" AS BOOLEAN) AS "extra_bool"', 'CAST("extra_int" AS BIGINT) AS "extra_int"', 'CAST("extra_float" AS DOUBLE) AS "extra_float"', - 'CAST("extra_decimal" AS DOUBLE) AS "extra_decimal"', + 'CAST("extra_decimal" AS DECIMAL(3, 2)) AS "extra_decimal"', 'CAST("extra_string" AS VARCHAR) AS "extra_string"', 'CAST("extra_timestamp" AS TIMESTAMP) AS "extra_timestamp"', 'CAST("extra_date" AS DATE) AS "extra_date"', diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 37ff868ee8..1d5452d345 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -229,8 +229,18 @@ GRANT SELECT ON system.query_log TO janitor_user; When usage can't be checked, because the adapter has no source or the role can't read it, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the -janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that -in-use relations are archived too. +janitor archives as usual. + +With `keep_in_use` on, the janitor needs the usage of every relation to archive. When usage can't +be checked, or the check leaves something out, such as view chains it couldn't follow, it stops +with `C503` and archives nothing: + +``` +error[C503]: keep_in_use needs the usage of every relation to archive, but usage was not checked: unsupported adapter; nothing was archived +``` + +Grant the role the access in the table above, or run without `keep_in_use` to archive relations +whether or not they are read. See [What counts as a read](#what-counts-as-a-read) for each warehouse's limits. diff --git a/website/src/content/docs/docs/concepts/diff.mdx b/website/src/content/docs/docs/concepts/diff.mdx index 93e4825fd3..2c0b87aff0 100644 --- a/website/src/content/docs/docs/concepts/diff.mdx +++ b/website/src/content/docs/docs/concepts/diff.mdx @@ -138,6 +138,10 @@ sqb diff prod:dev --full --select order_lines \ For a model without a stable row key, opt in to exact full-row multiset comparison with `--unkeyed`. Duplicate rows retain their multiplicity; SQLBuild never guesses a key. +An append model, or a microbatch `delete_insert` model, uses its `unique_key` only to match rows +here and in review. Neither replaces earlier rows by key, so the same key can appear more than once +in the table. See [incremental strategies](/docs/concepts/incremental#strategies). + ```bash sqb diff prod:dev --full --select daily_order_totals --unkeyed ``` diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index 50acb867cc..befff2abba 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -47,9 +47,13 @@ When `append_cursor_inclusive` is `true` (the default), the lower bound uses `>= Append without a cursor is also valid. The model simply inserts all rows from the source query on every run. +Append keeps every row. A `unique_key` on an append model only matches rows in +[diff](/docs/concepts/diff#row-matching) and [review](/docs/cli/review), so the same key can appear in the +table more than once. + ### delete_insert -Deletes rows in the cursor range, then inserts the new delta. Requires either `cursor` or `unique_key`. +Deletes the rows the new delta replaces, then inserts the delta. Requires either `cursor` or `unique_key`. ```sql MODEL ( @@ -65,7 +69,17 @@ SELECT order_id, customer_id, order_status, ordered_at, line_total_cents FROM __ref("fact_orders") ``` -With a cursor, `delete_insert` removes rows where the cursor column falls within the replay window, then inserts the new delta. With a `unique_key` only, it deletes matching rows by key before inserting. +Which rows it deletes depends on the model's settings: + +| Settings | Rows deleted before the insert | +|----------|--------------------------------| +| `cursor` | Rows whose cursor falls in the replay window | +| `unique_key` | Rows whose key appears in the delta | +| `cursor` and `unique_key` | Rows in the replay window, and rows whose key appears in the delta | +| `incremental_mode microbatch` | Rows in the batch's window; a `unique_key` only matches rows in diff and review | + +With both a cursor and a key, a row whose cursor value moved out of the window, such as an order +updated after its first load, replaces its earlier version instead of adding a second row. ### merge diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index e4ca878c5b..82b54e369f 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -226,7 +226,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged | `cursor_future_max_distance` | Largest distance a discovered cursor watermark may sit in the future, or `disabled` | | `cursor_future_action` | `cap` or `error` when a watermark exceeds `cursor_future_max_distance` | | `cursor_inputs` | Upstream names mapped to cursor columns | -| `unique_key` | Merge or delete/insert matching columns | +| `unique_key` | Row identity. `merge` and `delete_insert` replace rows with a matching key; `append` and microbatch `delete_insert` use it only to match rows in diff and review | | `incremental_mode` | Set to `microbatch` for batched execution | | `microbatch_strategy` | Required in microbatch mode: `watermark` or `rolling_window` | | `cursor_watermark_mode` | Watermark strategy policy: `all` or `any` | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 33a7d51f30..6fbf4a276e 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -115,9 +115,14 @@ explicitly: database = ":memory:" ``` -A `database` or `schema` on a connection doesn't decide where models are built; set them on the -target. A target that names an unknown connection fails when the configuration loads, before -anything connects. +A `database` or `schema` on a connection, named or a top-level `[connection]`, doesn't decide where +models are built; set them on the target or in `[defaults]`. Managed loaders and the janitor use +the same location as models: the target's `database` and `schema`, then `[defaults]`. On +Snowflake, BigQuery and Databricks they never use the connection's values. On the other adapters, +the connection's values fill in when neither the target nor `[defaults]` sets one. + +A target that names an unknown connection fails when the configuration loads, before anything +connects. ## Targets @@ -589,7 +594,7 @@ keep_in_use = false | `exclude_patterns` | `[]` | Glob patterns for relations to skip | | `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, and how many builds of each model and seed to keep in `_sqlbuild_consumption` | | `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](/docs/cli/janitor#relations-in-use) | -| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | +| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run. Stops with `C503` when usage can't be read in full | See [janitor](/docs/cli/janitor) for the archive and delete lifecycle. diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 508c2c89c4..9719cc75c5 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -60,6 +60,34 @@ Models reference managed sources the same way as any other source: SELECT id, name FROM __source("raw_customers") ``` +## Column types + +A column declared in the source's `columns` with a `type` gets that type, whatever Python values the +loader returns for it. SQLBuild infers the type of every other column from its values: + +| Python value | Column type | +|--------------|-------------| +| `bool` | Boolean | +| `int` | 64-bit integer | +| `float` | Double | +| `Decimal` | Decimal with the precision and scale of the column's values, up to 38 digits | +| `str` | String | +| `datetime` | Timestamp | +| `date` | Date | +| `dict` or `list` | JSON | + +For example, `Decimal("12.50")` and `Decimal("1234.5")` in one column give `DECIMAL(6, 2)`, written +in the adapter's own type, such as `NUMBER(6, 2)` on Snowflake. + +Details: + +- A value of any other type, such as a `UUID`, `bytes` or `time`, stops the load with an error + naming the column and the type. Declare the column's `type`, or convert the value first. +- Values of different types in one undeclared column, such as `float` and `Decimal`, stop the load. +- SQLBuild writes rows in batches of `load_batch_size` rows (10,000 by default), and a decimal + column's width comes from the first batch that has it. A later value that needs more digits stops + the load instead of being rounded; declare the column's `type` to allow it. + ## Write strategies The `write_strategy` field controls how returned rows are written to the target table. From 21762071168b15fc833c97eaa13844de0c48a5d5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:33:16 +0000 Subject: [PATCH 02/11] test(duckshop): unit test for the keyed cursor delete_insert edge model --- ...test_int_edge_delivery_status_replaced.sql | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_replaced.sql diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_replaced.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_replaced.sql new file mode 100644 index 0000000000..6e9fc1ccbf --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_status_replaced.sql @@ -0,0 +1,49 @@ +TEST ( + name "int_edge_delivery_status_replaced__copies_only_shipments_loaded_in_the_window", + cursor_start "2026-03-02 00:00:00", + cursor_end "2026-03-03 00:00:00", +); + +WITH __ref__int_edge_delivery_status AS ( + SELECT + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v.is_deleted, + v.latest_seq::NUMBER(38, 0) AS latest_seq, + v.source_loaded_at::TIMESTAMP_NTZ AS source_loaded_at + FROM VALUES + ('SH-1', 'DS-O-3001', 'in_transit', '2026-02-27', NULL, FALSE, 12, '2026-03-02 10:00:00'), + ('SH-3', 'DS-O-3003', 'delivered', '2026-02-20', '2026-02-25', FALSE, 30, '2026-03-01 10:00:00'), + ('SH-4', 'DS-O-3004', 'in_transit', '2026-02-26', NULL, TRUE, 41, '2026-03-02 23:59:59'), + ('SH-6', 'DS-O-3006', 'label_created', '2026-03-02', NULL, FALSE, 60, '2026-03-03 00:00:00') AS v( + shipment_id, + order_ref, + delivery_status, + ordered_on, + delivered_on, + is_deleted, + latest_seq, + source_loaded_at + ) +), + +__expected__int_edge_delivery_status_replaced AS ( + SELECT + 'SH-1' AS shipment_id, + 'in_transit' AS delivery_status, + FALSE AS is_deleted, + 12 AS latest_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'SH-4' AS shipment_id, + 'in_transit' AS delivery_status, + TRUE AS is_deleted, + 41 AS latest_seq, + '2026-03-02 23:59:59'::TIMESTAMP_NTZ(9) AS source_loaded_at +) + +SELECT 1 From 1a15d30f732f8ae62de3ee61d68c3c59c52168ed Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:35:57 +0000 Subject: [PATCH 03/11] refactor: drop MotherDuck's copy of the inherited loader type mapping --- .../motherduck/classes/motherduck_adapter.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 468d32013a..16e687ca85 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -32,27 +32,6 @@ class MotherDuckAdapter( adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value sql_analysis_dialect_name: ClassVar[str | None] = "duckdb" - def render_loader_logical_type( - self, *, type_name: LoaderLogicalType, precision: int = 38, scale: int = 0 - ) -> str: - match type_name: - case LoaderLogicalType.BOOLEAN: - return "BOOLEAN" - case LoaderLogicalType.INTEGER: - return "BIGINT" - case LoaderLogicalType.FLOAT: - return "DOUBLE" - case LoaderLogicalType.DECIMAL: - return f"DECIMAL({precision}, {scale})" - case LoaderLogicalType.STRING: - return "VARCHAR" - case LoaderLogicalType.TIMESTAMP: - return "TIMESTAMP" - case LoaderLogicalType.DATE: - return "DATE" - case LoaderLogicalType.JSON: - return "JSON" - def render_loader_value_literal( self, *, value: object, logical_type: LoaderLogicalType | None ) -> str: From a71c23738ba07dffe6e5d40d85d42ac003ba9ce6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:04:52 +0000 Subject: [PATCH 04/11] fix: keep keyed cursor writes atomic, decimals exact and loader locations consistent --- .../bindings/_helpers/load/load_selection.rs | 40 ++++++---- .../sqlbuild/references/docs/cli/janitor.md | 17 +++- .../sqlbuild/references/docs/concepts/diff.md | 4 + .../references/docs/concepts/incremental.md | 18 ++++- .../docs/concepts/models/configuration.md | 2 +- .../docs/concepts/project-configuration.md | 15 +++- .../docs/concepts/python-nodes/loaders.md | 29 +++++++ .../adapter/contract/classes/base_adapter.py | 7 +- .../contract/classes/duckdb_backed_adapter.py | 4 +- .../bigquery/classes/bigquery_adapter.py | 16 +++- .../clickhouse/classes/clickhouse_adapter.py | 4 +- .../databricks/classes/databricks_adapter.py | 19 +++-- src/sqlbuild/adapters/databricks/constants.py | 1 + .../motherduck/classes/motherduck_adapter.py | 4 +- .../postgres/classes/postgres_adapter.py | 4 +- .../snowflake/classes/snowflake_adapter.py | 27 +++++-- .../adapters/sqlite/classes/sqlite_adapter.py | 7 +- .../sqlserver/classes/sqlserver_adapter.py | 4 +- .../sqlbuild/adapters/sqlite/_test_types.py | 7 ++ .../sqlbuild/adapters/sqlite/test_client.py | 41 ++++++++++ .../cli/commands/main/test_load_command.py | 32 +++++++- .../sqlbuild/adapters/bigquery/_test_types.py | 17 ++++ .../sqlbuild/adapters/bigquery/test_client.py | 78 +++++++++++++++++++ .../adapters/databricks/_test_types.py | 8 ++ .../sqlbuild/adapters/databricks/helpers.py | 8 ++ .../adapters/databricks/test_client.py | 54 ++++++++++++- .../adapters/snowflake/_test_types.py | 8 ++ .../sqlbuild/adapters/snowflake/helpers.py | 15 +++- .../test_keyed_cursor_delete_insert.py | 68 ++++++++++++++++ .../_helpers/load_selection/_test_types.py | 1 + .../_helpers/load_selection/test_selection.py | 24 ++++++ website/src/content/docs/docs/cli/janitor.mdx | 3 +- .../docs/concepts/project-configuration.mdx | 8 +- 33 files changed, 537 insertions(+), 57 deletions(-) create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index b20cc9d2a4..209c7c0bd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -15,6 +15,8 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; +use crate::bindings::_helpers::compile_attachment::target::project_location; +use crate::bindings::_helpers::compile_driver::targets::requires_explicit_targets; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::host; @@ -230,23 +232,18 @@ fn non_empty_text<'py>(value: Option>) -> PyResult( +/// The target's value for `field`, or `None` when there is no target. +fn target_value<'py>( target_config: &Bound<'py, PyAny>, field: &str, - fallback: Option>, ) -> PyResult> { - let py: Python<'py> = target_config.py(); - if !target_config.is_none() { - let value: Bound<'py, PyAny> = native_field(target_config, field)?; - if value.is_truthy()? { - return Ok(value); - } + if target_config.is_none() { + return Ok(target_config.clone()); } - Ok(fallback.unwrap_or_else(|| py.None().into_bound(py))) + native_field(target_config, field) } -/// Intermediate loader defaults: the target namespace, else the connection's (no database on DuckDB). +/// Intermediate loader defaults: the project location models use (see `project_location`). pub(crate) fn loader_defaults<'py>( discovered: &Bound<'py, PyAny>, (selected_target, target_config, cli_vars): ( @@ -274,13 +271,22 @@ pub(crate) fn loader_defaults<'py>( } else { non_empty_text(connection.get_item("database")?)? }; + let explicit: bool = requires_explicit_targets(&adapter_name)?; + let defaults: Bound<'py, PyAny> = + native_field(&native_field(discovered, "project_config")?, "defaults")?; + let location = |field: &str, fallback: Option>| { + project_location( + ( + target_value(target_config, field)?, + native_field(&defaults, field)?, + ), + fallback.into_iter().collect(), + explicit, + ) + }; Ok(LoaderDefaults { - database: first_truthy(target_config, "database", connection_database)?, - schema: first_truthy( - target_config, - "schema", - non_empty_text(connection.get_item("schema")?)?, - )?, + database: location("database", connection_database)?, + schema: location("schema", non_empty_text(connection.get_item("schema")?)?)?, }) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md index fffdcaa3b3..f30f489492 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/janitor.md @@ -243,8 +243,18 @@ GRANT SELECT ON system.query_log TO janitor_user; When usage can't be checked, because the adapter has no source or the role can't read it, the preview and the prompt show one notice, such as `usage not checked: unsupported adapter`, and the -janitor archives as usual. With `keep_in_use` on, a second notice, `nothing kept in use`, says that -in-use relations are archived too. +janitor archives as usual. + +With `keep_in_use` on, the janitor needs the usage of every relation to archive. When usage can't +be checked, or the check leaves something out, such as view chains it couldn't follow, it stops +with `C503` and archives nothing: + +``` +error[C503]: keep_in_use needs the usage of every relation to archive, but usage was not checked: unsupported adapter; nothing was archived +``` + +Grant the role the access in the table above, or run without `keep_in_use` to archive relations +whether or not they are read. See [What counts as a read](#what-counts-as-a-read) for each warehouse's limits. @@ -394,7 +404,8 @@ missed or don't count: - queries on other replicas or shards, which matters on ClickHouse Cloud and behind load-balanced replicas. When `system.clusters` shows other replicas of the connected server's shard, the preview - says `usage read from this server only; other replicas not checked`; + says `usage read from this server only; other replicas not checked`, and `keep_in_use` stops + with `C503` because the usage is incomplete; - queries ClickHouse answers from metadata without logging the table, such as a plain `count()`; - readers whose settings profile turns `log_queries` off or samples it with `log_queries_probability`. When `log_queries` is off for the janitor's own connection, usage isn't diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md index 36f118de52..8f6d19b5ea 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/diff.md @@ -150,6 +150,10 @@ sqb diff prod:dev --full --select order_lines \ For a model without a stable row key, opt in to exact full-row multiset comparison with `--unkeyed`. Duplicate rows retain their multiplicity; SQLBuild never guesses a key. +An append model, or a microbatch `delete_insert` model, uses its `unique_key` only to match rows +here and in review. Neither replaces earlier rows by key, so the same key can appear more than once +in the table. See [incremental strategies](incremental.md#strategies). + ```bash sqb diff prod:dev --full --select daily_order_totals --unkeyed ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 0846575b22..a0ea0cf0da 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -58,9 +58,13 @@ When `append_cursor_inclusive` is `true` (the default), the lower bound uses `>= Append without a cursor is also valid. The model simply inserts all rows from the source query on every run. +Append keeps every row. A `unique_key` on an append model only matches rows in +[diff](diff.md#row-matching) and [review](../cli/review.md), so the same key can appear in the +table more than once. + ### delete_insert -Deletes rows in the cursor range, then inserts the new delta. Requires either `cursor` or `unique_key`. +Deletes the rows the new delta replaces, then inserts the delta. Requires either `cursor` or `unique_key`. ```sql MODEL ( @@ -76,7 +80,17 @@ SELECT order_id, customer_id, order_status, ordered_at, line_total_cents FROM __ref("fact_orders") ``` -With a cursor, `delete_insert` removes rows where the cursor column falls within the replay window, then inserts the new delta. With a `unique_key` only, it deletes matching rows by key before inserting. +Which rows it deletes depends on the model's settings: + +| Settings | Rows deleted before the insert | +|----------|--------------------------------| +| `cursor` | Rows whose cursor falls in the replay window | +| `unique_key` | Rows whose key appears in the delta | +| `cursor` and `unique_key` | Rows in the replay window, and rows whose key appears in the delta | +| `incremental_mode microbatch` | Rows in the batch's window; a `unique_key` only matches rows in diff and review | + +With both a cursor and a key, a row whose cursor value moved out of the window, such as an order +updated after its first load, replaces its earlier version instead of adding a second row. ### merge diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index f1eaafc8d9..5213641a9f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -242,7 +242,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged | `cursor_future_max_distance` | Largest distance a discovered cursor watermark may sit in the future, or `disabled` | | `cursor_future_action` | `cap` or `error` when a watermark exceeds `cursor_future_max_distance` | | `cursor_inputs` | Upstream names mapped to cursor columns | -| `unique_key` | Merge or delete/insert matching columns | +| `unique_key` | Row identity. `merge` and `delete_insert` replace rows with a matching key; `append` and microbatch `delete_insert` use it only to match rows in diff and review | | `incremental_mode` | Set to `microbatch` for batched execution | | `microbatch_strategy` | Required in microbatch mode: `watermark` or `rolling_window` | | `cursor_watermark_mode` | Watermark strategy policy: `all` or `any` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 1d1b1add4a..4cd9906e68 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -140,9 +140,16 @@ explicitly: database = ":memory:" ``` -A `database` or `schema` on a connection doesn't decide where models are built; set them on the -target. A target that names an unknown connection fails when the configuration loads, before -anything connects. +A `database` or `schema` on a connection, named or a top-level `[connection]`, doesn't decide where +models are built; set them on the target or in `[defaults]`. Intermediate loaders and the janitor +use the same location as models: the target's `database` and `schema`, then `[defaults]`. On +Snowflake, BigQuery and Databricks they never use the connection's values. On the other adapters, +the connection's values fill in when neither the target nor `[defaults]` sets one. Managed sources +load into their own `database` and `schema`, then the target's, as described in +[Targets](#targets). + +A target that names an unknown connection fails when the configuration loads, before anything +connects. ## Targets @@ -614,7 +621,7 @@ keep_in_use = false | `exclude_patterns` | `[]` | Glob patterns for relations to skip | | `direct_state_history_versions` | `20` | How many state-history rows to keep per identity in `_sqlbuild_fingerprints` and `_sqlbuild_source_freshness`, and how many builds of each model and seed to keep in `_sqlbuild_consumption` | | `in_use_window` | `"7d"` | How far back the janitor looks for reads of each relation it would archive. A fixed duration such as `"7d"` or `"36h"`. See [Relations in use](../cli/janitor.md#relations-in-use) | -| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run | +| `keep_in_use` | `false` | Keep relations read within `in_use_window` instead of archiving them. `--keep-in-use` turns it on for one run. Stops with `C503` when usage can't be read in full | See [janitor](../cli/janitor.md) for the archive and delete lifecycle. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index 46c9fdb96e..cd462648a0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/loaders/ - How it works - Defining a loader - Binding to a source +- Column types - Write strategies - Self-managed loaders - Loader context @@ -79,6 +80,34 @@ Models reference managed sources the same way as any other source: SELECT id, name FROM __source("raw_customers") ``` +## Column types + +A column declared in the source's `columns` with a `type` gets that type, whatever Python values the +loader returns for it. SQLBuild infers the type of every other column from its values: + +| Python value | Column type | +|--------------|-------------| +| `bool` | Boolean | +| `int` | 64-bit integer | +| `float` | Double | +| `Decimal` | Decimal with the precision and scale of the column's values, up to 38 digits | +| `str` | String | +| `datetime` | Timestamp | +| `date` | Date | +| `dict` or `list` | JSON | + +For example, `Decimal("12.50")` and `Decimal("1234.5")` in one column give `DECIMAL(6, 2)`, written +in the adapter's own type, such as `NUMBER(6, 2)` on Snowflake. + +Details: + +- A value of any other type, such as a `UUID`, `bytes` or `time`, stops the load with an error + naming the column and the type. Declare the column's `type`, or convert the value first. +- Values of different types in one undeclared column, such as `float` and `Decimal`, stop the load. +- SQLBuild writes rows in batches of `load_batch_size` rows (10,000 by default), and a decimal + column's width comes from the first batch that has it. A later value that needs more digits stops + the load instead of being rounded; declare the column's `type` to allow it. + ## Write strategies The `write_strategy` field controls how returned rows are written to the target table. diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index e1d7cb17e8..f0c625af14 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -1514,7 +1514,8 @@ def delete_insert_cursor( stmt: str for stmt in statements: cursor: Any = self.execute(connection=connection, sql=stmt) - affected = self.affected_row_count(cursor=cursor) + if stmt.startswith("INSERT"): + affected = self.affected_row_count(cursor=cursor) return affected def merge( @@ -2053,7 +2054,9 @@ def render_loader_value_literal( return "NULL" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return _quote_sql_string(value.isoformat()) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 476b47d91e..9060b04c15 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -454,7 +454,9 @@ def render_loader_value_literal( return "NULL" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return self._quote_sql_string(value.isoformat()) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 6aac74eaff..0505314342 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1136,7 +1136,9 @@ def render_loader_value_literal( return f"JSON {self._quote_sql_string(json.dumps(value, sort_keys=True))}" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return self._quote_sql_string(format(value, "f")) + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return self._quote_sql_string(value.isoformat()) @@ -1820,6 +1822,8 @@ def render_delete_insert_cursor( cursor_type: str | None = None, unique_key: tuple[str, ...] = (), ) -> tuple[str, ...]: + """Replace a cursor window; with a key and columns, one MERGE also replaces delta keys.""" + insert_clause: str = "INSERT ROW" if columns is not None: column_list: str = ", ".join(self.render_identifier(column) for column in columns) @@ -1833,6 +1837,14 @@ def render_delete_insert_cursor( f"AND __target.{self.render_identifier(cursor_column)} < " f"{self._render_cursor_bound_string(cursor_end)}" ) + if unique_key and columns is not None: + (merge_sql,) = self.render_merge( + destination=self._quote_identifier_path(destination), + sql=sql, + unique_key=unique_key, + source_columns=columns, + ) + return (f"{merge_sql} WHEN NOT MATCHED BY SOURCE AND {cursor_filter} THEN DELETE",) if unique_key: return ( f"DELETE FROM {self._quote_identifier_path(destination)} AS __target " @@ -3114,6 +3126,8 @@ def _to_bigquery_type(column_type: str) -> str: return DATETIME_TYPE_NAME if any(token in normalized for token in ("FLOAT", "DOUBLE", "REAL")): return FLOAT_WIRE_TYPE_NAME + if any(token in normalized for token in ("BIGNUMERIC", "BIGDECIMAL")): + return "BIGNUMERIC" if any(token in normalized for token in ("DECIMAL", "NUMERIC", "NUMBER")): return "NUMERIC" if INTEGER_TYPE_TOKEN in normalized: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 74bd7d7ae7..711272e944 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -580,7 +580,9 @@ def render_loader_value_literal( return quote_string(json.dumps(value, sort_keys=True)) if isinstance(value, bool): return "true" if value else "false" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return quote_string(format(value, "f")) + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return quote_string(value.isoformat()) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 8bda13a764..30354664b9 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -136,6 +136,7 @@ DETAILED_TABLE_INFORMATION_MARKER, NON_ROW_RESULT_COLUMN_NAMES, RELATION_NOT_FOUND_ERROR_CLASSES, + SERVER_OPERATION_ERROR_CLASS_NAME, STRING_LITERAL_ESCAPES, TABLE_RELATION_METADATA_TYPES, TEMPORARY_FUNCTION_PREFIX, @@ -698,6 +699,10 @@ def _get_delta_history_freshness_metadata( cursor.execute(" UNION ALL ".join(selects)) rows: list[tuple[Any, ...]] = list(cursor.fetchall()) except Exception as error: + if not any( + kind.__name__ == SERVER_OPERATION_ERROR_CLASS_NAME for kind in type(error).__mro__ + ): + raise if len(requests) == 1: reason: str = (str(error).strip().splitlines() or [type(error).__name__])[0] return { @@ -860,7 +865,9 @@ def render_loader_value_literal( return "NULL" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return self._quote_sql_string(value.isoformat()) @@ -1819,11 +1826,13 @@ def render_delete_insert_cursor( f"{quoted_cursor_column} >= {start_bound} AND {quoted_cursor_column} < {end_bound}" ) if unique_key: + key_delete, *insert_statements = self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns + ) + key_match: str = key_delete.removeprefix(f"DELETE FROM {destination} WHERE ") return ( - f"DELETE FROM {destination} WHERE {replace_where}", - *self.render_delete_insert( - destination=destination, sql=sql, unique_key=unique_key, columns=columns - ), + f"DELETE FROM {destination} WHERE ({replace_where}) OR {key_match}", + *insert_statements, ) return (f"INSERT INTO {destination}{column_list} REPLACE WHERE {replace_where} {sql}",) diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index dcf2f31f21..e90a625426 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -14,6 +14,7 @@ "NO_SUCH_CATALOG_EXCEPTION", ) UNDEFINED_TABLE_SQLSTATE: str = "42P01" +SERVER_OPERATION_ERROR_CLASS_NAME: str = "ServerOperationError" DETAILED_TABLE_INFORMATION_MARKER: str = "# detailed table information" DESCRIBE_DETAIL_ROW_WIDTH: int = 2 TEMPORARY_FUNCTION_PREFIX: str = "__sqb_test__" diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 16e687ca85..c86c73acef 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -39,7 +39,9 @@ def render_loader_value_literal( return "NULL" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return self._quote_sql_string(value.isoformat()) diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 321e6d566d..442ec02f9d 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -2020,7 +2020,9 @@ def render_loader_value_literal( return f"{self._quote_sql_string(json.dumps(value, sort_keys=True))}::JSONB" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return self._quote_sql_string(value.isoformat()) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 22bfa43afa..4780bbf52e 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -1200,12 +1200,11 @@ def render_delete_insert_cursor( f"AND {quoted_cursor_column} < {self._quote_sql_string(cursor_end)}" ) if unique_key: - return ( - delete_sql, - *self.render_delete_insert( - destination=destination, sql=sql, unique_key=unique_key, columns=columns - ), + keyed: tuple[str, ...] = self.render_delete_insert( + destination=destination, sql=sql, unique_key=unique_key, columns=columns ) + staged: int = sum(1 for statement in keyed if statement.startswith("CREATE ")) + return (*keyed[:staged], delete_sql, *keyed[staged:]) insert_stmts: tuple[str, ...] = self.render_append( destination=destination, sql=sql, columns=columns ) @@ -1583,6 +1582,8 @@ def delete_insert_cursor( cursor_type: str | None = None, unique_key: tuple[str, ...] = (), ) -> int | None: + """Replace a cursor window; staging DDL runs outside the transaction, which DDL commits.""" + statements: tuple[str, ...] = self.render_delete_insert_cursor( destination=destination, sql=sql, @@ -1594,8 +1595,18 @@ def delete_insert_cursor( unique_key=unique_key, ) statement_recorder.record_many(statements) + staged: tuple[str, ...] = tuple(s for s in statements if s.startswith("CREATE ")) + dropped: tuple[str, ...] = tuple(s for s in statements if s.startswith("DROP ")) + for statement in staged: + self.execute(connection=connection, sql=statement) with self.transaction(connection): - return self._inserted_rows(connection=connection, statements=statements) + inserted: int | None = self._inserted_rows( + connection=connection, + statements=tuple(s for s in statements if s not in staged and s not in dropped), + ) + for statement in dropped: + self.execute(connection=connection, sql=statement) + return inserted def _inserted_rows(self, *, connection: Any, statements: tuple[str, ...]) -> int | None: """Run DML statements; the rows the INSERT among them wrote, as Snowflake reports them.""" @@ -1677,7 +1688,9 @@ def render_loader_value_literal( return "NULL" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return self._quote_sql_string(value.isoformat()) diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index d28382af60..e8759a3ed2 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -1288,7 +1288,8 @@ def delete_insert_cursor( stmt: str for stmt in statements: cursor: Any = self.execute(connection=connection, sql=stmt) - affected = self.affected_row_count(cursor=cursor) + if stmt.startswith(INSERT_KEYWORD): + affected = self.affected_row_count(cursor=cursor) return affected def merge( @@ -2291,7 +2292,9 @@ def render_loader_value_literal( return _quote_sql_string(json.dumps(value, sort_keys=True)) if isinstance(value, bool): return "1" if value else "0" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime): return _quote_sql_string(canonical_timestamp_text(value.isoformat())) diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index c279286f0d..26c30be512 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -1018,7 +1018,9 @@ def render_loader_value_literal( return self._quote_sql_string(json.dumps(value, sort_keys=True)) if isinstance(value, bool): return "1" if value else "0" - if isinstance(value, int | float | Decimal): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): return self._quote_sql_string(value.isoformat()) diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py index 64b140c19c..a9a0f5d037 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py @@ -226,3 +226,10 @@ class SqliteStatisticsTestCase: description: str fixture_name: str expected_interactive_row_count: int + + +@dataclass(frozen=True) +class SqliteCursorRowCountTestCase: + description: str + unique_key: tuple[str, ...] + expected_inserted: int diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py index b9310a1a5a..c7d0fa346b 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -19,6 +20,7 @@ SqliteAtomicityTestCase, SqliteCastTestCase, SqliteConnectErrorTestCase, + SqliteCursorRowCountTestCase, SqliteDiffTestCase, SqliteNamespaceListingTestCase, SqlitePragmaTestCase, @@ -830,3 +832,42 @@ def test_given_sqlite_rows_when_sampling_metadata_then_unknown_without_scanning( assert interactive.row_count == test_case.expected_interactive_row_count assert interactive.detail is None + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCursorRowCountTestCase( + description="a keyed window reports the insert's rows, not its staging drop", + unique_key=("order_id",), + expected_inserted=1, + ), + SqliteCursorRowCountTestCase( + description="an unkeyed window reports the insert's rows", + unique_key=(), + expected_inserted=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_cursor_window_when_writing_then_sqlite_reports_inserted_rows( + test_case: SqliteCursorRowCountTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + + inserted: int | None = adapter.delete_insert_cursor( + connection=connection, + destination="orders", + sql=( + "SELECT 1 AS order_id, 10 AS customer_id, 13 AS amount, " + "'2026-01-02 15:00:00' AS ordered_at, 'shipped' AS status" + ), + cursor_column="ordered_at", + cursor_start="2026-01-02 00:00:00", + cursor_end="2026-01-03 00:00:00", + cursor_type="timestamp", + statement_recorder=StatementRecorder(), + unique_key=test_case.unique_key, + ) + + assert inserted == test_case.expected_inserted diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 6aa83438cf..22b9c2b2db 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -3959,6 +3959,36 @@ def raw_inferred(ctx): expected_row=("12345678-1234-5678-1234-567812345678", Decimal("12.50")), expected_column_types={"order_ref": "VARCHAR", "price": "DECIMAL(6,2)"}, ), + LoadCommandInferredColumnsTestCase( + description="wide decimals stay exact beside values written with an exponent", + project_files={ + "sqlbuild_project.toml": _PROJECT_FILE, + "sources/raw.yml": """ +sources: + - name: raw_inferred + description: Test source raw_inferred. + managed: true + write_strategy: table + columns: + - name: order_ref + type: VARCHAR +""".strip() + + "\n", + "python/loaders/raw.py": """ +from decimal import Decimal + +from sqlbuild.loaders import loader + +@loader +def raw_inferred(ctx): + '''Test loader raw_inferred.''' + yield {"order_ref": "a_wide", "price": Decimal("12345678901234567890.123")} + yield {"order_ref": "b_tiny", "price": Decimal("1E-7")} +""", + }, + expected_row=("a_wide", Decimal("12345678901234567890.123")), + expected_column_types={"order_ref": "VARCHAR", "price": "DECIMAL(27,7)"}, + ), ], ids=lambda case: case.description, ) @@ -3977,7 +4007,7 @@ def test_given_decimal_and_declared_values_when_running_load_then_keeps_their_ty connection: DuckDBPyConnection = duckdb.connect(str(tmp_path / "demo.duckdb")) try: row: tuple[object, ...] | None = connection.execute( - "SELECT order_ref, price FROM raw_inferred ORDER BY price LIMIT 1" + "SELECT order_ref, price FROM raw_inferred ORDER BY order_ref NULLS LAST LIMIT 1" ).fetchone() column_rows: list[tuple[str, str]] = connection.execute( "SELECT column_name, data_type FROM information_schema.columns " diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 471a294490..dfc21e8e21 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -1,6 +1,7 @@ from collections.abc import Callable from dataclasses import dataclass from datetime import datetime +from decimal import Decimal from sqlbuild._native import InferredNullability from sqlbuild.adapter.contract.models import ( @@ -282,3 +283,19 @@ class BigQueryGoogleSqlRenderingTestCase: description: str render: Callable[[BigQueryAdapter], str | tuple[str, ...]] expected_sql: str | tuple[str, ...] + + +@dataclass(frozen=True) +class BigQueryKeyedCursorMergeTestCase: + description: str + unique_key: tuple[str, ...] + columns: tuple[str, ...] + expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class BigQueryLoaderDecimalRowTestCase: + description: str + value: Decimal + column_type: str + expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index 19cccaa5c1..6dc1e797ca 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta, timezone +from decimal import Decimal import pytest @@ -22,6 +23,7 @@ from sqlbuild.adapter.contract.types import ( CursorKind, FunctionNullabilityRule, + LoaderLogicalType, TableFreshnessStatus, ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -39,6 +41,8 @@ BigQueryDmlLifecycleTestCase, BigQueryExecutionErrorTestCase, BigQueryExpressionInferenceProfileTestCase, + BigQueryKeyedCursorMergeTestCase, + BigQueryLoaderDecimalRowTestCase, BigQueryMergeExclusionTestCase, BigQueryPruneSqlTestCase, BigQueryQueryTestCase, @@ -895,6 +899,47 @@ def test_given_delete_insert_cursor_when_rendering_then_bigquery_uses_merge( assert unexpected_fragment not in rendered_sql +@pytest.mark.parametrize( + "test_case", + [ + BigQueryKeyedCursorMergeTestCase( + description="one merge replaces the delta keys and deletes the rest of the window", + unique_key=("order_id",), + columns=("order_id", "status", "updated_at"), + expected_statements=( + "MERGE INTO `example-project.dev.orders` AS __target USING " + "(SELECT order_id, status, updated_at FROM delta_orders) AS __source " + "ON __target.`order_id` = __source.`order_id` " + "WHEN MATCHED THEN UPDATE SET `status` = __source.`status`, " + "`updated_at` = __source.`updated_at` " + "WHEN NOT MATCHED THEN INSERT (`order_id`, `status`, `updated_at`) " + "VALUES (__source.`order_id`, __source.`status`, __source.`updated_at`) " + "WHEN NOT MATCHED BY SOURCE AND __target.`updated_at` >= " + "TIMESTAMP '2026-01-01T00:00:00' AND __target.`updated_at` < " + "TIMESTAMP '2026-01-02T00:00:00' THEN DELETE", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_keyed_cursor_window_when_rendering_then_bigquery_replaces_it_in_one_merge( + test_case: BigQueryKeyedCursorMergeTestCase, +) -> None: + adapter: BigQueryAdapter = BigQueryAdapter() + + statements: tuple[str, ...] = adapter.render_delete_insert_cursor( + destination="example-project.dev.orders", + sql="SELECT order_id, status, updated_at FROM delta_orders", + cursor_column="updated_at", + cursor_start="2026-01-01T00:00:00", + cursor_end="2026-01-02T00:00:00", + columns=test_case.columns, + unique_key=test_case.unique_key, + ) + + assert statements == test_case.expected_statements + + @pytest.mark.parametrize( "test_case", [ @@ -1152,3 +1197,36 @@ def test_given_listed_relations_when_adding_age_metadata_then_bigquery_reads_eac "SELECT table_id, TIMESTAMP_MILLIS(creation_time), " "TIMESTAMP_MILLIS(last_modified_time) FROM `example-project.dev_orders.__TABLES__`", ) + + +@pytest.mark.parametrize( + "test_case", + [ + BigQueryLoaderDecimalRowTestCase( + description="a wide decimal in exponent form casts exactly to bignumeric", + value=Decimal("1.23456789012E-2"), + column_type="BIGNUMERIC(40, 13)", + expected_sql="SELECT CAST('0.0123456789012' AS BIGNUMERIC) AS `amount`", + ), + BigQueryLoaderDecimalRowTestCase( + description="a numeric decimal casts from a string, not a float literal", + value=Decimal("12.50"), + column_type="NUMERIC(4, 2)", + expected_sql="SELECT CAST('12.50' AS NUMERIC) AS `amount`", + ), + ], + ids=lambda case: case.description, +) +def test_given_decimal_loader_value_when_rendering_rows_then_bigquery_casts_it_exactly( + test_case: BigQueryLoaderDecimalRowTestCase, +) -> None: + adapter: BigQueryAdapter = BigQueryAdapter() + + rendered_sql: str = adapter.render_loader_rows_select( + rows=({"amount": test_case.value},), + column_names=("amount",), + column_sql_types={"amount": test_case.column_type}, + inferred_types={"amount": LoaderLogicalType.DECIMAL}, + ) + + assert rendered_sql == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index 29ef30218c..d43d91fcbc 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -59,6 +59,7 @@ class DatabricksRenderDeleteInsertCursorTestCase: cursor_end: str columns: tuple[str, ...] | None expected_statements: tuple[str, ...] + unique_key: tuple[str, ...] = () @dataclass(frozen=True) @@ -201,3 +202,10 @@ class DatabricksSchemaExistsTestCase: rows: list[tuple[object, ...]] expected_exists: bool expected_query_fragment: str + + +@dataclass(frozen=True) +class DatabricksFreshnessBatchErrorTestCase: + description: str + batch_error: Exception + expected_match: str diff --git a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py index 98643e29dd..1dae5347e6 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/helpers.py @@ -45,6 +45,14 @@ def close(self) -> None: self.closed = True +class ServerOperationError(Exception): + """Stand-in named like the connector's error for a statement the server rejected.""" + + +class RequestError(Exception): + """Stand-in named like the connector's error for a failed request to the server.""" + + class FailingDatabricksMetadataCursor(FakeDatabricksMetadataCursor): def __init__(self, *, execute_error: Exception) -> None: super().__init__() diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index dc39e824a0..4e53389c01 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -19,6 +19,7 @@ from tests.unit.src.sqlbuild.adapters.databricks._test_types import ( DatabricksDeltaDetailAgeTestCase, DatabricksExpressionInferenceProfileTestCase, + DatabricksFreshnessBatchErrorTestCase, DatabricksMergeExclusionTestCase, DatabricksPruneSqlTestCase, DatabricksPythonFunctionSupportTestCase, @@ -37,6 +38,8 @@ FailingDatabricksMetadataCursor, FakeDatabricksMetadataConnection, FakeDatabricksMetadataCursor, + RequestError, + ServerOperationError, ) @@ -263,13 +266,13 @@ def test_given_view_in_freshness_batch_when_getting_metadata_then_only_the_view_ database="main", schema="raw", name="orders_v" ) batch_cursor: FakeDatabricksMetadataCursor = FailingDatabricksMetadataCursor( - execute_error=RuntimeError(test_case.view_error) + execute_error=ServerOperationError(test_case.view_error) ) orders_cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor( rows=[("main", "raw", "orders", test_case.delta_version)] ) view_cursor: FakeDatabricksMetadataCursor = FailingDatabricksMetadataCursor( - execute_error=RuntimeError(test_case.view_error) + execute_error=ServerOperationError(test_case.view_error) ) connection: FakeDatabricksMetadataConnection = FakeDatabricksMetadataConnection( (batch_cursor, orders_cursor, view_cursor) @@ -292,6 +295,34 @@ def test_given_view_in_freshness_batch_when_getting_metadata_then_only_the_view_ ) +@pytest.mark.parametrize( + "test_case", + [ + DatabricksFreshnessBatchErrorTestCase( + description="a lost connection fails the check instead of marking sources unknown", + batch_error=RequestError("connection reset"), + expected_match="connection reset", + ) + ], + ids=lambda case: case.description, +) +def test_given_request_error_in_freshness_batch_when_getting_metadata_then_it_is_raised( + test_case: DatabricksFreshnessBatchErrorTestCase, +) -> None: + adapter: DatabricksAdapter = DatabricksAdapter() + requests: tuple[TableFreshnessRequest, ...] = ( + TableFreshnessRequest(database="main", schema="raw", name="orders"), + TableFreshnessRequest(database="main", schema="raw", name="customers"), + ) + connection: FakeDatabricksMetadataConnection = FakeDatabricksMetadataConnection( + (FailingDatabricksMetadataCursor(execute_error=test_case.batch_error),) + ) + + with pytest.raises(type(test_case.batch_error), match=test_case.expected_match): + adapter.get_tables_freshness_metadata(connection=connection, requests=requests) + assert len(connection.returned_cursors) == 1 + + @pytest.mark.parametrize( "test_case", [ @@ -324,6 +355,24 @@ def test_given_view_in_freshness_batch_when_getting_metadata_then_only_the_view_ "SELECT id, status FROM `workspace`.`test`.`orders__delta`", ), ), + DatabricksRenderDeleteInsertCursorTestCase( + description="keyed cursor deletes the window and the delta keys in one statement", + target="`workspace`.`test`.`orders`", + sql="SELECT id, status FROM `workspace`.`test`.`orders__delta`", + cursor_column="id", + cursor_start="1", + cursor_end="10", + columns=("id", "status"), + unique_key=("id",), + expected_statements=( + "DELETE FROM `workspace`.`test`.`orders` WHERE (`id` >= 1 AND `id` < 10) " + "OR EXISTS (SELECT 1 FROM (SELECT id, status FROM " + "`workspace`.`test`.`orders__delta`) AS __source " + "WHERE `workspace`.`test`.`orders`.`id` = __source.`id`)", + "INSERT INTO `workspace`.`test`.`orders` (`id`, `status`) " + "SELECT id, status FROM `workspace`.`test`.`orders__delta`", + ), + ), ], ids=lambda case: case.description, ) @@ -339,6 +388,7 @@ def test_given_cursor_delete_insert_when_rendering_then_databricks_uses_replace_ cursor_start=test_case.cursor_start, cursor_end=test_case.cursor_end, columns=test_case.columns, + unique_key=test_case.unique_key, ) assert statements == test_case.expected_statements diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index e06ff737f7..a984ec19a2 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -426,3 +426,11 @@ class SnowflakeSchemaExistsTestCase: cursor: object expected_exists: bool expected_sql_fragment: str + + +@dataclass(frozen=True) +class SnowflakeKeyedCursorDeleteInsertTestCase: + description: str + unique_key: tuple[str, ...] + expected_prefixes: tuple[str, ...] + expected_inserted: int diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index 0c6d24b4f4..8b15802d82 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from io import StringIO from itertools import cycle, islice -from types import ModuleType +from types import ModuleType, SimpleNamespace from typing import Any from sqlbuild._native import NativeProgressProjector @@ -682,3 +682,16 @@ def prefix_listing_statements(warehouse: RecordingSnowflakeWarehouse) -> tuple[s sql.split(" IN SCHEMA ")[0] for sql in filter(lambda sql: sql.startswith("SHOW TERSE OBJECTS"), warehouse.attempted_sql) ) + + +class RecordingSnowflakeAdapter(SnowflakeAdapter): + """Snowflake adapter that records each statement instead of running it.""" + + def __init__(self) -> None: + super().__init__() + self.executed: list[str] = [] + + def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any: + del connection, kwargs + self.executed.append(sql) + return SimpleNamespace(rowcount=3) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py new file mode 100644 index 0000000000..ca2daab813 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_keyed_cursor_delete_insert.py @@ -0,0 +1,68 @@ +"""Keyed cursor delete_insert on Snowflake keeps its deletes and insert in one transaction.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeKeyedCursorDeleteInsertTestCase, +) +from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( + RecordingSnowflakeAdapter, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeKeyedCursorDeleteInsertTestCase( + description="DDL that commits stays outside the transaction around the deletes", + unique_key=("order_id",), + expected_prefixes=( + "CREATE", + "BEGIN", + "DELETE FROM ANALYTICS.MARTS.ORDERS WHERE", + "DELETE FROM ANALYTICS.MARTS.ORDERS USING", + "INSERT", + "COMMIT", + "DROP", + ), + expected_inserted=3, + ), + SnowflakeKeyedCursorDeleteInsertTestCase( + description="an unkeyed window keeps its delete and insert in one transaction", + unique_key=(), + expected_prefixes=( + "BEGIN", + "DELETE FROM ANALYTICS.MARTS.ORDERS WHERE", + "INSERT", + "COMMIT", + ), + expected_inserted=3, + ), + ], + ids=lambda case: case.description, +) +def test_given_cursor_window_when_replacing_then_snowflake_commits_deletes_with_insert( + test_case: SnowflakeKeyedCursorDeleteInsertTestCase, +) -> None: + adapter: RecordingSnowflakeAdapter = RecordingSnowflakeAdapter() + + inserted: int | None = adapter.delete_insert_cursor( + connection=object(), + destination="ANALYTICS.MARTS.ORDERS", + sql="SELECT * FROM ANALYTICS.MARTS.ORDERS__DELTA", + cursor_column="updated_at", + cursor_start="2026-01-02", + cursor_end="2026-01-03", + statement_recorder=StatementRecorder(), + unique_key=test_case.unique_key, + ) + + assert len(adapter.executed) == len(test_case.expected_prefixes) + assert all( + statement.startswith(prefix) + for statement, prefix in zip(adapter.executed, test_case.expected_prefixes, strict=True) + ) + assert inserted == test_case.expected_inserted diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py index abcfcc1182..baf23eeade 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py @@ -29,3 +29,4 @@ class LoaderDefaultResolutionTestCase: adapter: str connection: dict[str, object] expected_defaults: tuple[str | None, str | None] + project_defaults: tuple[str | None, str | None] = (None, None) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index de12ddeff6..ce4fc1efe2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + DefaultsConfig, DiscoveredLoaderFunction, DiscoveredProjectInputs, LocalConfig, @@ -139,6 +140,26 @@ def test_given_intermediate_loader_when_selecting_then_parses_destination_with_e connection={"database": "connection_db", "schema": "connection_schema"}, expected_defaults=("connection_db", "connection_schema"), ), + LoaderDefaultResolutionTestCase( + description="project defaults come before the connection namespace", + adapter="postgres", + connection={"database": "connection_db", "schema": "connection_schema"}, + project_defaults=("defaults_db", "defaults_schema"), + expected_defaults=("defaults_db", "defaults_schema"), + ), + LoaderDefaultResolutionTestCase( + description="explicit-target adapters never take the connection namespace", + adapter="snowflake", + connection={"database": "CONNECTION_DB", "schema": "CONNECTION_SCHEMA"}, + expected_defaults=(None, None), + ), + LoaderDefaultResolutionTestCase( + description="explicit-target adapters take the project defaults", + adapter="snowflake", + connection={"database": "CONNECTION_DB", "schema": "CONNECTION_SCHEMA"}, + project_defaults=("DEFAULTS_DB", "DEFAULTS_SCHEMA"), + expected_defaults=("DEFAULTS_DB", "DEFAULTS_SCHEMA"), + ), ), ids=lambda case: case.description, ) @@ -150,6 +171,9 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co name="demo", adapter=test_case.adapter, connection=test_case.connection, + defaults=DefaultsConfig( + database=test_case.project_defaults[0], schema=test_case.project_defaults[1] + ), ), local_config=LocalConfig(), ) diff --git a/website/src/content/docs/docs/cli/janitor.mdx b/website/src/content/docs/docs/cli/janitor.mdx index 1d5452d345..179f61647e 100644 --- a/website/src/content/docs/docs/cli/janitor.mdx +++ b/website/src/content/docs/docs/cli/janitor.mdx @@ -390,7 +390,8 @@ missed or don't count: - queries on other replicas or shards, which matters on ClickHouse Cloud and behind load-balanced replicas. When `system.clusters` shows other replicas of the connected server's shard, the preview - says `usage read from this server only; other replicas not checked`; + says `usage read from this server only; other replicas not checked`, and `keep_in_use` stops + with `C503` because the usage is incomplete; - queries ClickHouse answers from metadata without logging the table, such as a plain `count()`; - readers whose settings profile turns `log_queries` off or samples it with `log_queries_probability`. When `log_queries` is off for the janitor's own connection, usage isn't diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 6fbf4a276e..cbaab24139 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -116,10 +116,12 @@ database = ":memory:" ``` A `database` or `schema` on a connection, named or a top-level `[connection]`, doesn't decide where -models are built; set them on the target or in `[defaults]`. Managed loaders and the janitor use -the same location as models: the target's `database` and `schema`, then `[defaults]`. On +models are built; set them on the target or in `[defaults]`. Intermediate loaders and the janitor +use the same location as models: the target's `database` and `schema`, then `[defaults]`. On Snowflake, BigQuery and Databricks they never use the connection's values. On the other adapters, -the connection's values fill in when neither the target nor `[defaults]` sets one. +the connection's values fill in when neither the target nor `[defaults]` sets one. Managed sources +load into their own `database` and `schema`, then the target's, as described in +[Targets](#targets). A target that names an unknown connection fails when the configuration loads, before anything connects. From 3d3154d2d295b4e6cac1740e7627c7a1b3c58571 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:05:58 +0000 Subject: [PATCH 05/11] refactor: drop MotherDuck's copy of the inherited loader literal rendering --- .../motherduck/classes/motherduck_adapter.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index c86c73acef..ed78457cd2 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -2,9 +2,6 @@ from __future__ import annotations -import json -from datetime import date, datetime -from decimal import Decimal from typing import Any, ClassVar from urllib.parse import urlencode @@ -32,23 +29,6 @@ class MotherDuckAdapter( adapter_name: ClassVar[str] = BuiltinAdapter.MOTHERDUCK.value sql_analysis_dialect_name: ClassVar[str | None] = "duckdb" - def render_loader_value_literal( - self, *, value: object, logical_type: LoaderLogicalType | None - ) -> str: - if value is None: - return "NULL" - if isinstance(value, bool): - return "TRUE" if value else "FALSE" - if isinstance(value, Decimal): - return format(value, "f") - if isinstance(value, int | float): - return str(value) - if isinstance(value, datetime | date): - return self._quote_sql_string(value.isoformat()) - if isinstance(value, dict | list): - return self._quote_sql_string(json.dumps(value, sort_keys=True)) - return self._quote_sql_string(str(value)) - def render_loader_rows_select( self, *, From 12c1374dd661ee6ecaf99d75283f96ba8904b90f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:10:01 +0000 Subject: [PATCH 06/11] fix: quote loader decimals only for inferred decimal columns --- .../adapters/bigquery/classes/bigquery_adapter.py | 4 +++- .../adapters/clickhouse/classes/clickhouse_adapter.py | 4 +++- .../unit/src/sqlbuild/adapters/bigquery/_test_types.py | 4 ++++ .../unit/src/sqlbuild/adapters/bigquery/test_client.py | 10 ++++++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 0505314342..c8ae2ec073 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1136,8 +1136,10 @@ def render_loader_value_literal( return f"JSON {self._quote_sql_string(json.dumps(value, sort_keys=True))}" if isinstance(value, bool): return "TRUE" if value else "FALSE" - if isinstance(value, Decimal): + if isinstance(value, Decimal) and logical_type == LoaderLogicalType.DECIMAL: return self._quote_sql_string(format(value, "f")) + if isinstance(value, Decimal): + return format(value, "f") if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 711272e944..854c772a33 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -580,8 +580,10 @@ def render_loader_value_literal( return quote_string(json.dumps(value, sort_keys=True)) if isinstance(value, bool): return "true" if value else "false" - if isinstance(value, Decimal): + if isinstance(value, Decimal) and logical_type == LoaderLogicalType.DECIMAL: return quote_string(format(value, "f")) + if isinstance(value, Decimal): + return format(value, "f") if isinstance(value, int | float): return str(value) if isinstance(value, datetime | date): diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index dfc21e8e21..604430fdb0 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -11,6 +11,7 @@ RowDiffSampleRow, SchemaDiffResult, ) +from sqlbuild.adapter.contract.types import LoaderLogicalType from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -299,3 +300,6 @@ class BigQueryLoaderDecimalRowTestCase: value: Decimal column_type: str expected_sql: str + inferred_types: tuple[tuple[str, LoaderLogicalType], ...] = ( + ("amount", LoaderLogicalType.DECIMAL), + ) diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index 6dc1e797ca..e4dcf6a1e4 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -23,7 +23,6 @@ from sqlbuild.adapter.contract.types import ( CursorKind, FunctionNullabilityRule, - LoaderLogicalType, TableFreshnessStatus, ) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -1214,6 +1213,13 @@ def test_given_listed_relations_when_adding_age_metadata_then_bigquery_reads_eac column_type="NUMERIC(4, 2)", expected_sql="SELECT CAST('12.50' AS NUMERIC) AS `amount`", ), + BigQueryLoaderDecimalRowTestCase( + description="a decimal in a declared integer column stays a numeric literal", + value=Decimal("3.00"), + column_type="INT64", + inferred_types=(), + expected_sql="SELECT CAST(3.00 AS INT64) AS `amount`", + ), ], ids=lambda case: case.description, ) @@ -1226,7 +1232,7 @@ def test_given_decimal_loader_value_when_rendering_rows_then_bigquery_casts_it_e rows=({"amount": test_case.value},), column_names=("amount",), column_sql_types={"amount": test_case.column_type}, - inferred_types={"amount": LoaderLogicalType.DECIMAL}, + inferred_types=dict(test_case.inferred_types), ) assert rendered_sql == test_case.expected_sql From 245d51278bef633fff4a888b747859f222b01104 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:06:02 +0000 Subject: [PATCH 07/11] fix: treat a lone top-level connection as the default target and name ignored connection locations --- .../_helpers/compile_attachment/target.rs | 50 ++--- .../compile_driver/assembly_project.rs | 16 +- .../_helpers/compile_driver/project.rs | 21 +- .../_helpers/compile_driver/targets.rs | 181 ++++++++++++------ .../bindings/_helpers/load/load_selection.rs | 20 +- fensu.toml | 2 +- .../docs/concepts/project-configuration.md | 33 +++- .../cli/commands/main/compile/_test_types.py | 9 + .../compile/test_missing_target_database.py | 94 ++++++++- .../cli/commands/main/sqlite/test_sqlite.py | 3 +- .../_helpers/load_selection/_test_types.py | 3 +- .../_helpers/load_selection/test_selection.py | 10 +- .../compile/_helpers/test_assembly.py | 78 +++++++- .../docs/concepts/project-configuration.mdx | 33 +++- 14 files changed, 419 insertions(+), 134 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 9cbe3f8877..34a83cb59a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -3,15 +3,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::compile_attachment::effective_config::project_vars; use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_CACHE_DIRECTORY_NAMES, }; -use crate::bindings::_helpers::compile_driver::targets::requires_explicit_targets; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; @@ -124,27 +121,31 @@ fn optional_string<'py>( }) } -/// A project-level location as models resolve it: target, then `[defaults]`, then any fallback. +/// A project-level location: the target's value, then `[defaults]`, then the first set fallback. pub(crate) fn project_location<'py>( (target, default): (Bound<'py, PyAny>, Bound<'py, PyAny>), fallbacks: Vec>, - explicit: bool, ) -> PyResult> { let py: Python<'py> = target.py(); - if target.is_truthy()? && !target.eq(PRESERVE_TARGET_VALUE)? { - return Ok(target); - } - if default.is_truthy()? { - return Ok(default); + for value in [target, default].into_iter().chain(fallbacks) { + if value.is_truthy()? { + return Ok(value); + } } - if !explicit { - for fallback in fallbacks { - if fallback.is_truthy()? { - return Ok(fallback); + Ok(py.None().into_bound(py)) +} + +/// Whether the top-level `[connection]` is the implicit default target: no named connections or targets. +pub(crate) fn connection_is_default_target(discovered: &Bound<'_, PyAny>) -> PyResult { + for config in ["project_config", "local_config"] { + let config: Bound<'_, PyAny> = native_field(discovered, config)?; + for field in ["connections", "targets"] { + if native_field(&config, field)?.is_truthy()? { + return Ok(false); } } } - Ok(py.None().into_bound(py)) + Ok(true) } /// Resolve the target and its effective database/schema without compiling resources. @@ -186,10 +187,7 @@ pub(crate) fn build_effective_target_namespace<'py>( }; let project_config: Bound<'py, PyAny> = native_field(discovered_inputs, "project_config")?; let defaults: Bound<'py, PyAny> = native_field(&project_config, "defaults")?; - let explicit: bool = requires_explicit_targets(&resolve_effective_adapter_name( - &project_config, - &native_field(discovered_inputs, "local_config")?, - )?)?; + let inline: bool = connection_is_default_target(discovered_inputs)?; let values: Bound<'py, PyDict> = PyDict::new(py); values.set_item( DATABASE_KEY, @@ -198,8 +196,11 @@ pub(crate) fn build_effective_target_namespace<'py>( target_value(DATABASE_KEY)?, native_field(&defaults, DATABASE_KEY)?, ), - vec![connection_database, default_database], - explicit, + [connection_database] + .into_iter() + .filter(|_| inline) + .chain([default_database]) + .collect(), )?, )?; values.set_item( @@ -209,8 +210,11 @@ pub(crate) fn build_effective_target_namespace<'py>( target_value(SCHEMA_KEY)?, native_field(&defaults, SCHEMA_KEY)?, ), - vec![connection_schema, default_schema], - explicit, + [connection_schema] + .into_iter() + .filter(|_| inline) + .chain([default_schema]) + .collect(), )?, )?; let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index faf1c2f1d0..221320816a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -10,7 +10,9 @@ use sqlbuild_analysis::assembly::project::main::configured_namespace::configured use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::keywords; -use crate::bindings::_helpers::compile_attachment::target::project_location; +use crate::bindings::_helpers::compile_attachment::target::{ + connection_is_default_target, project_location, +}; use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, analysis_context, and_not, @@ -24,7 +26,6 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compile_driver::stage_work::{SEMANTIC_COMPLETION, count_stage}; -use crate::bindings::_helpers::compile_driver::targets::requires_explicit_targets; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::contracts::spec_resolution::{ resolve_effective_adapter_name, resolve_effective_scenario_config, @@ -179,22 +180,23 @@ fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult = native_field(inputs, "effective_connection")?; let defaults: Bound<'py, PyAny> = native_field(&native_field(inputs, "project_config")?, "defaults")?; - let explicit: bool = requires_explicit_targets(&effective_adapter_name(inputs)?)?; + let inline: bool = connection_is_default_target(&native_field(inputs, "discovered_inputs")?)?; + let connection_value = |value: Bound<'py, PyAny>| -> Vec> { + [value].into_iter().filter(|_| inline).collect() + }; let database: Bound<'py, PyAny> = project_location( ( target_field(inputs, "database")?, native_field(&defaults, "database")?, ), - vec![connection_database_fallback(inputs, &connection)?], - explicit, + connection_value(connection_database_fallback(inputs, &connection)?), )?; let schema: Bound<'py, PyAny> = project_location( ( target_field(inputs, "schema")?, native_field(&defaults, "schema")?, ), - vec![str_or_none(&mapping_get(&connection, "schema")?)?], - explicit, + connection_value(str_or_none(&mapping_get(&connection, "schema")?)?), )?; early_model_templates( keywords(py, &[("database", database), ("schema", schema)])?.as_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index cb0e58f3e3..8c6a9ca065 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::compile_driver::target_defaults::{ }; use crate::bindings::_helpers::compile_driver::targets::{ managed_loader_target_isolation, managed_write_schemas, named_target_schema_strategy, - project_targets, + project_targets, requires_explicit_targets, }; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; use crate::bindings::_helpers::records::construct::{enum_member, new_record_with}; @@ -216,19 +216,26 @@ fn targeted_project<'py>( } else { adapter.call_method0("default_schema")? }; + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; + let adapter_name: Bound<'py, PyAny> = + resolve_effective_adapter_name(&project_config, &local_config.clone()) + .and_then(|value| value.into_bound_py_any(py))?; + let effective_database: Bound<'py, PyAny> = native_field(project, "effective_target_database")?; + let default_database: Bound<'py, PyAny> = + if requires_explicit_targets(&adapter_name)? && effective_database.is_truthy()? { + effective_database + } else { + adapter.call_method0("default_database")? + }; let defaults: TargetDefaults<'py> = TargetDefaults { default_schema, - default_database: adapter.call_method0("default_database")?, + default_database, render_qualified_name: native_field(adapter, "render_qualified_name")?, python_functions_inherit_default_namespace: adapter .call_method0("python_functions_inherit_default_namespace")?, }; let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; - let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; - let adapter_name: Bound<'py, PyAny> = - resolve_effective_adapter_name(&project_config, &local_config.clone()) - .and_then(|value| value.into_bound_py_any(py))?; project_targets(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 9171a9aafd..708a5372aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -39,6 +39,7 @@ const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databricks"]; const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; +const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; @@ -100,6 +101,51 @@ fn text_or_none(value: &Bound<'_, PyAny>) -> PyResult> { } } +/// The connection's `part` when the compile leaves it unused because the project names connections or targets. +fn ignored_connection_value(project: &Bound<'_, PyAny>, part: &str) -> PyResult> { + if text_or_none(&native_field(project, &format!("effective_target_{part}"))?)?.is_some() { + return Ok(None); + } + text_or_none(&mapping_get( + &native_field(project, "effective_connection")?, + part, + )?) +} + +/// The sentence saying the connection's `part` is unused. +fn ignored_connection_note(part: &str, value: &str) -> String { + format!( + " The connection's {part} '{value}' is not used because the project defines named \ + connections or targets." + ) +} + +/// `setting_help` for adding `key = "value"` under `[section]` of the project file. +fn project_setting_help( + py: Python<'_>, + (purpose, section): (&str, &str), + (key, value): (&str, &str), +) -> PyResult { + setting_help( + py, + Some(keywords( + py, + &[ + ("purpose", PyString::new(py, purpose).into_any()), + ( + "file_name", + PyString::new(py, PROJECT_CONFIG_FILENAME).into_any(), + ), + ("section", PyString::new(py, section).into_any()), + ("key", PyString::new(py, key).into_any()), + ("value", PyString::new(py, value).into_any()), + ], + )?), + ) + .and_then(|value| value.into_bound_py_any(py))? + .extract() +} + /// Require shared-warehouse named targets to choose a schema strategy. #[pyfunction] #[pyo3(signature = (**options))] @@ -127,9 +173,6 @@ pub(crate) fn named_target_schema_strategy( selected_target: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = discovered.py(); - if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? { - return Ok(()); - } let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; let target_name: Bound<'_, PyAny> = resolve_target_name( @@ -149,14 +192,25 @@ pub(crate) fn named_target_schema_strategy( if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); } - Err(planner_error( - py, - target_error_message(&TargetError::NamedTargetSchema( - target_name.str()?.to_cow()?.into_owned(), - )), - NAMED_TARGET_SCHEMA_CODE, - None, - )) + let name: String = target_name.str()?.to_cow()?.into_owned(); + let mut message: String = target_error_message(&TargetError::NamedTargetSchema(name.clone())); + let connection: Bound<'_, PyAny> = + build_effective_connection_config(discovered, Some(target_name), None, false)?; + let help: Option = match text_or_none(&mapping_get(&connection, "schema")?)? { + None if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? => { + return Ok(()); + } + Some(value) => { + message.push_str(&ignored_connection_note("schema", &value)); + Some(project_setting_help( + py, + (NAMED_TARGET_HELP_PURPOSE, &format!("targets.{name}")), + ("schema", &value), + )?) + } + None => None, + }; + Err(planner_error(py, message, NAMED_TARGET_SCHEMA_CODE, help)) } /// Reject managed loader write namespaces shared by multiple targets. @@ -425,17 +479,23 @@ pub(crate) fn project_targets( } } for (resource_name, target) in targets { - required_target_parts((&adapter_label, resource_kind), resource_name, &target)?; + required_target_parts( + (&adapter_label, resource_kind), + resource_name, + &target, + project, + )?; } } Ok(()) } -/// Raise when `target` misses its database or schema. +/// Raise when `target` misses its database or schema, naming any connection value left unused. fn required_target_parts( (adapter_name, resource_kind): (&str, &str), resource_name: String, target: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = target.py(); let database: Option = optional_text(&native_field(target, "database")?)?; @@ -444,39 +504,30 @@ fn required_target_parts( let Some(first) = missing.first() else { return Ok(()); }; - let help: String = setting_help( + let mut message: String = target_error_message(&TargetError::MissingParts(MissingParts { + adapter_name: adapter_name.to_owned(), + resource_kind: resource_kind.to_owned(), + resource_name, + database, + schema, + })); + let mut first_value: String = format!("my_{first}"); + for part in &missing { + if let Some(value) = ignored_connection_value(project, part)? { + message.push_str(&ignored_connection_note(part, &value)); + if part == first { + first_value = value; + } + } + } + let help: String = project_setting_help( py, - Some(keywords( - py, - &[ - ( - "purpose", - PyString::new(py, DEFAULTS_HELP_PURPOSE).into_any(), - ), - ( - "file_name", - PyString::new(py, PROJECT_CONFIG_FILENAME).into_any(), - ), - ("section", PyString::new(py, DEFAULTS_SECTION).into_any()), - ("key", PyString::new(py, first).into_any()), - ( - "value", - PyString::new(py, &format!("my_{first}")).into_any(), - ), - ], - )?), - ) - .and_then(|value| value.into_bound_py_any(py))? - .extract()?; + (DEFAULTS_HELP_PURPOSE, DEFAULTS_SECTION), + (first, &first_value), + )?; Err(planner_error( py, - target_error_message(&TargetError::MissingParts(MissingParts { - adapter_name: adapter_name.to_owned(), - resource_kind: resource_kind.to_owned(), - resource_name, - database, - schema, - })), + message, MISSING_TARGET_PARTS_CODE, Some(help), )) @@ -503,14 +554,14 @@ pub(crate) fn managed_write_schemas( adapter: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult<()> { - if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? { + let ignored: Option = ignored_connection_value(project, "schema")?; + if ignored.is_none() + && native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? + { return Ok(()); } - let connection_schema: Option = explicit_write_schema( - text_or_none( - &native_field(project, "effective_connection")?.call_method1("get", ("schema",))?, - )? - .as_deref(), + let project_schema: Option = explicit_write_schema( + text_or_none(&native_field(project, "effective_target_schema")?)?.as_deref(), ); for (label, attribute) in [ ("Model", "models"), @@ -525,7 +576,7 @@ pub(crate) fn managed_write_schemas( native_field(&resource, "name")?.str()?.to_cow()? ), &native_field(&resource, "destination")?.getattr("schema")?, - connection_schema.as_deref(), + (project_schema.as_deref(), ignored.as_deref()), )?; } } @@ -541,29 +592,37 @@ pub(crate) fn managed_write_schemas( native_field(&source, "name")?.str()?.to_cow()? ), &native_field(&entry, "schema")?, - connection_schema.as_deref(), + (project_schema.as_deref(), ignored.as_deref()), )?; } Ok(()) } -/// Raise when neither the resource nor the connection names a physical write schema. +/// Raise when neither the resource nor the project names a physical write schema. fn managed_write_schema( resource_label: &str, schema: &Bound<'_, PyAny>, - connection_schema: Option<&str>, + (project_schema, ignored): (Option<&str>, Option<&str>), ) -> PyResult<()> { - if explicit_write_schema(text_or_none(schema)?.as_deref()).is_some() - || connection_schema.is_some() + if explicit_write_schema(text_or_none(schema)?.as_deref()).is_some() || project_schema.is_some() { return Ok(()); } - Err(planner_error( - schema.py(), - target_error_message(&TargetError::ManagedWriteSchema(resource_label.to_owned())), - MANAGED_WRITE_SCHEMA_CODE, - None, - )) + let py: Python<'_> = schema.py(); + let mut message: String = + target_error_message(&TargetError::ManagedWriteSchema(resource_label.to_owned())); + let help: Option = match ignored { + Some(value) => { + message.push_str(&ignored_connection_note("schema", value)); + Some(project_setting_help( + py, + (DEFAULTS_HELP_PURPOSE, DEFAULTS_SECTION), + ("schema", value), + )?) + } + None => None, + }; + Err(planner_error(py, message, MANAGED_WRITE_SCHEMA_CODE, help)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 209c7c0bd9..02d076e9db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_core::load_command::errors::UnknownLoadSelector; use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; @@ -15,8 +16,9 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; -use crate::bindings::_helpers::compile_attachment::target::project_location; -use crate::bindings::_helpers::compile_driver::targets::requires_explicit_targets; +use crate::bindings::_helpers::compile_attachment::target::{ + connection_is_default_target, project_location, +}; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::host; @@ -271,17 +273,21 @@ pub(crate) fn loader_defaults<'py>( } else { non_empty_text(connection.get_item("database")?)? }; - let explicit: bool = requires_explicit_targets(&adapter_name)?; + let inline: bool = connection_is_default_target(discovered)?; let defaults: Bound<'py, PyAny> = native_field(&native_field(discovered, "project_config")?, "defaults")?; - let location = |field: &str, fallback: Option>| { + let location = |field: &str, connection_value: Option>| { + let target: Bound<'py, PyAny> = target_value(target_config, field)?; project_location( ( - target_value(target_config, field)?, + if target.eq(PRESERVE_TARGET_VALUE)? { + py.None().into_bound(py) + } else { + target + }, native_field(&defaults, field)?, ), - fallback.into_iter().collect(), - explicit, + connection_value.into_iter().filter(|_| inline).collect(), ) }; Ok(LoaderDefaults { diff --git a/fensu.toml b/fensu.toml index 0d1892628f..e7cd8dcc61 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2013,7 +2013,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 4cd9906e68..3c98c3c3c4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -140,13 +140,32 @@ explicitly: database = ":memory:" ``` -A `database` or `schema` on a connection, named or a top-level `[connection]`, doesn't decide where -models are built; set them on the target or in `[defaults]`. Intermediate loaders and the janitor -use the same location as models: the target's `database` and `schema`, then `[defaults]`. On -Snowflake, BigQuery and Databricks they never use the connection's values. On the other adapters, -the connection's values fill in when neither the target nor `[defaults]` sets one. Managed sources -load into their own `database` and `schema`, then the target's, as described in -[Targets](#targets). +Where models build depends on how the project names its connection: + +| Project shape | Models, intermediate loaders and the janitor use | +|---------------|--------------------------------------------------| +| One top-level `[connection]`, no `[connections.*]` and no `[targets]` | `[defaults]`, then the connection's `database` and `schema` | +| Named `[connections.*]` or any `[targets]` | The target's `database` and `schema`, then `[defaults]`; never the connection's | + +A project with a single top-level `[connection]` treats it as its default target, so this is +enough on every adapter: + +```toml +name = "orders" +adapter = "snowflake" + +[connection] +account = "my_org-my_account" +user = "${ENV:SNOWFLAKE_USER}" +password = "${ENV:SNOWFLAKE_PASSWORD}" +database = "ANALYTICS" +schema = "ORDERS" +``` + +Once the project names connections or targets, a `database` or `schema` on a connection is +ignored, and the `S101`, `S103` or `S104` error for a missing location names the ignored value and +the setting to add. Managed sources load into their own `database` and `schema`, then the +target's, as described in [Targets](#targets). A target that names an unknown connection fails when the configuration loads, before anything connects. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 8ce02d930a..45f691e682 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1422,3 +1422,12 @@ class PythonMacroArgumentErrorTestCase: description: str call_sql: str expected_message: str + + +@dataclass(frozen=True) +class InlineConnectionLocationTestCase: + """A project whose only location is its inline connection, and where its models build.""" + + description: str + project_toml: str + expected_qualified_name: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 025221c974..c47390faab 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -1,13 +1,16 @@ -"""CLI e2e coverage: the S101 missing-database error and its single settings example.""" +"""CLI e2e coverage: missing-location errors name unused connection values and one setting.""" from __future__ import annotations +import json import subprocess from pathlib import Path +from typing import Any import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + InlineConnectionLocationTestCase, MissingTargetDatabaseTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -31,10 +34,48 @@ ), expected_example_count=1, ), + MissingTargetDatabaseTestCase( + description="a named connection's database is named as unused", + project_toml=( + 'name = "orders"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[connections.warehouse]\naccount = "example"\ndatabase = "ANALYTICS"\n\n' + '[targets.dev]\nschema = "orders"\n' + ), + expected_error=( + "error[S101]: snowflake execution requires explicit target database. model " + "'orders' resolved to database=None schema='orders'. Set them in the model " + "config, the environment config, or the project defaults. The connection's " + "database 'ANALYTICS' is not used because the project defines named connections " + "or targets.\n" + " = help: to set project-wide defaults, set this in sqlbuild_project.toml:\n" + " [defaults]\n" + ' database = "ANALYTICS"\n' + ), + expected_example_count=1, + ), + MissingTargetDatabaseTestCase( + description="a named connection's schema is named as unused", + project_toml=( + 'name = "orders"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[connections.warehouse]\naccount = "example"\nschema = "ORDERS"\n\n' + '[targets.dev]\ndatabase = "ANALYTICS"\n' + ), + expected_error=( + "error[S104]: Named target 'dev' must explicitly set schema to a nonblank literal " + "or 'preserve'. Resource schemas, defaults, and connection schemas do not select " + "a named target schema strategy. The connection's schema 'ORDERS' is not used " + "because the project defines named connections or targets.\n" + " = help: to build this target into that schema, set this in " + "sqlbuild_project.toml:\n" + " [targets.dev]\n" + ' schema = "ORDERS"\n' + ), + expected_example_count=0, + ), ), ids=lambda case: case.description, ) -def test_given_snowflake_model_without_database_when_compiling_then_s101_shows_one_example( +def test_given_snowflake_model_without_location_when_compiling_then_error_names_the_setting( test_case: MissingTargetDatabaseTestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( @@ -56,3 +97,52 @@ def test_given_snowflake_model_without_database_when_compiling_then_s101_shows_o assert result.returncode == 1, result.stdout + result.stderr assert test_case.expected_error in result.stderr assert result.stderr.count("[defaults]") == test_case.expected_example_count + + +@pytest.mark.parametrize( + "test_case", + ( + InlineConnectionLocationTestCase( + description="an inline connection database and schema place snowflake models", + project_toml=( + 'name = "orders"\nadapter = "snowflake"\n\n[connection]\naccount = "example"\n' + 'database = "ANALYTICS"\nschema = "ORDERS"\n' + ), + expected_qualified_name="ANALYTICS.ORDERS.orders", + ), + InlineConnectionLocationTestCase( + description="an inline connection schema places bigquery models beside defaults", + project_toml=( + 'name = "orders"\nadapter = "bigquery"\n\n[connection]\n' + 'project = "example-project"\nschema = "orders"\n\n' + '[defaults]\ndatabase = "example-project"\n' + ), + expected_qualified_name="`example-project.orders.orders`", + ), + ), + ids=lambda case: case.description, +) +def test_given_inline_connection_location_when_compiling_then_models_build_there( + test_case: InlineConnectionLocationTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": test_case.project_toml, + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized view);\n\n" + "SELECT 1 AS order_id\n" + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("compile", "--no-cache", "--json"), project_dir=project_dir + ) + payload: dict[str, Any] = json.loads(result.stdout) + + assert result.returncode == 0, f"{test_case.description}: {result.stdout}{result.stderr}" + assert [model["qualified_name"] for model in payload["resources"]["models"]] == [ + test_case.expected_qualified_name + ], test_case.description diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py index 0c5409bfe8..21f1a162ca 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py @@ -202,13 +202,12 @@ def test_given_cloned_target_when_diffing_then_sqlite_reports_changed_rows( absent_tables=("analytics___sqlbuild_microbatches", "_sqlbuild_microbatches"), ), SqliteProjectE2ETestCase( - description="cursor delete_insert replaces only the cursor window", + description="keyed cursor delete_insert replaces the window and changed keys", files=STRATEGY_PROJECT_FILES, changed_files={"seeds/order_events.csv": CHANGED_EVENTS_CSV}, query="SELECT * FROM analytics__orders_window ORDER BY 1, 3", expected_rows=( (1, 100, "2026-01-01 10:00:00"), - (2, 200, "2026-01-01 11:00:00"), (2, 250, "2026-01-02 12:00:00"), (3, 300, "2026-01-02 09:00:00"), (4, 400, "2026-01-03 08:00:00"), diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py index baf23eeade..53e20809fd 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/_test_types.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field @dataclass(frozen=True) @@ -30,3 +30,4 @@ class LoaderDefaultResolutionTestCase: connection: dict[str, object] expected_defaults: tuple[str | None, str | None] project_defaults: tuple[str | None, str | None] = (None, None) + connections: dict[str, dict[str, object]] = field(default_factory=dict) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index ce4fc1efe2..3bfc6126cc 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -148,9 +148,16 @@ def test_given_intermediate_loader_when_selecting_then_parses_destination_with_e expected_defaults=("defaults_db", "defaults_schema"), ), LoaderDefaultResolutionTestCase( - description="explicit-target adapters never take the connection namespace", + description="an inline connection is the default target on explicit-target adapters", adapter="snowflake", connection={"database": "CONNECTION_DB", "schema": "CONNECTION_SCHEMA"}, + expected_defaults=("CONNECTION_DB", "CONNECTION_SCHEMA"), + ), + LoaderDefaultResolutionTestCase( + description="a named connection's location is never used", + adapter="snowflake", + connection={}, + connections={"warehouse": {"database": "NAMED_DB", "schema": "NAMED_SCHEMA"}}, expected_defaults=(None, None), ), LoaderDefaultResolutionTestCase( @@ -171,6 +178,7 @@ def test_given_connection_namespace_when_resolving_load_defaults_then_matches_co name="demo", adapter=test_case.adapter, connection=test_case.connection, + connections=test_case.connections, defaults=DefaultsConfig( database=test_case.project_defaults[0], schema=test_case.project_defaults[1] ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 5a0d94f556..84f47a13cc 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -14,6 +14,7 @@ assemble_project, build_compile_inputs, discover_project_inputs, + resolve_loader_defaults, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.types import ( @@ -734,7 +735,7 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph "test_case", ( AssembleCompiledProjectEffectiveTargetTestCase( - description="falls back effective target location to connection location", + description="a target never takes its connection's location on duckdb", project_toml=""" name = "demo" adapter = "duckdb" @@ -746,10 +747,10 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph """.strip() + "\n", expected_effective_target_database=None, - expected_effective_target_schema="connection_schema", + expected_effective_target_schema=None, ), AssembleCompiledProjectEffectiveTargetTestCase( - description="falls back non duckdb target database to connection database", + description="a target never takes its connection's location on postgres", project_toml=""" name = "demo" adapter = "postgres" @@ -760,8 +761,8 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph schema = "connection_schema" """.strip() + "\n", - expected_effective_target_database="connection_db", - expected_effective_target_schema="connection_schema", + expected_effective_target_database=None, + expected_effective_target_schema=None, ), AssembleCompiledProjectEffectiveTargetTestCase( description="prefers target location over connection fallback", @@ -816,10 +817,59 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph expected_effective_target_database=None, expected_effective_target_schema=None, ), + AssembleCompiledProjectEffectiveTargetTestCase( + description="an inline connection is the implicit default target on snowflake", + project_toml=""" +name = "demo" +adapter = "snowflake" + +[connection] +database = "CONN_DB" +schema = "CONN_SCHEMA" +""".strip() + + "\n", + expected_effective_target_database="CONN_DB", + expected_effective_target_schema="CONN_SCHEMA", + ), + AssembleCompiledProjectEffectiveTargetTestCase( + description="an inline connection schema fills in beside a defaults database", + project_toml=""" +name = "demo" +adapter = "bigquery" + +[connection] +project = "example-project" +schema = "orders" + +[defaults] +database = "example-project" +""".strip() + + "\n", + expected_effective_target_database="example-project", + expected_effective_target_schema="orders", + ), + AssembleCompiledProjectEffectiveTargetTestCase( + description="a top-level connection beside named targets is not used", + project_toml=""" +name = "demo" +adapter = "snowflake" +default_target = "dev" + +[connection] +database = "CONN_DB" +schema = "CONN_SCHEMA" + +[targets.dev] +database = "TARGET_DB" +""".strip() + + "\n", + expected_effective_target_database="TARGET_DB", + expected_effective_target_schema=None, + ), ), ids=lambda case: case.description, ) -def test_given_connection_location_when_assembling_project_then_sets_effective_target_location( +def test_given_connection_location_when_assembling_project_then_models_and_loaders_agree( test_case: AssembleCompiledProjectEffectiveTargetTestCase, tmp_path: Path, write_repo_files: Callable[[Path, dict[str, str]], None], @@ -836,9 +886,21 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t ) compiled: CompiledProjectView = assemble_project(inputs=compile_inputs) + loader_location: tuple[str | None, str | None] = resolve_loader_defaults( + discovered_inputs=discovered, + selected_target=compiled.effective_target_name, + target_config=compile_inputs.effective_target, + ) - assert compiled.effective_target_database == test_case.expected_effective_target_database - assert compiled.effective_target_schema == test_case.expected_effective_target_schema + expected_location: tuple[str | None, str | None] = ( + test_case.expected_effective_target_database, + test_case.expected_effective_target_schema, + ) + assert ( + compiled.effective_target_database, + compiled.effective_target_schema, + ) == expected_location, test_case.description + assert loader_location == expected_location, f"{test_case.description}: loaders" @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index cbaab24139..db7cf7bce7 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -115,13 +115,32 @@ explicitly: database = ":memory:" ``` -A `database` or `schema` on a connection, named or a top-level `[connection]`, doesn't decide where -models are built; set them on the target or in `[defaults]`. Intermediate loaders and the janitor -use the same location as models: the target's `database` and `schema`, then `[defaults]`. On -Snowflake, BigQuery and Databricks they never use the connection's values. On the other adapters, -the connection's values fill in when neither the target nor `[defaults]` sets one. Managed sources -load into their own `database` and `schema`, then the target's, as described in -[Targets](#targets). +Where models build depends on how the project names its connection: + +| Project shape | Models, intermediate loaders and the janitor use | +|---------------|--------------------------------------------------| +| One top-level `[connection]`, no `[connections.*]` and no `[targets]` | `[defaults]`, then the connection's `database` and `schema` | +| Named `[connections.*]` or any `[targets]` | The target's `database` and `schema`, then `[defaults]`; never the connection's | + +A project with a single top-level `[connection]` treats it as its default target, so this is +enough on every adapter: + +```toml +name = "orders" +adapter = "snowflake" + +[connection] +account = "my_org-my_account" +user = "${ENV:SNOWFLAKE_USER}" +password = "${ENV:SNOWFLAKE_PASSWORD}" +database = "ANALYTICS" +schema = "ORDERS" +``` + +Once the project names connections or targets, a `database` or `schema` on a connection is +ignored, and the `S101`, `S103` or `S104` error for a missing location names the ignored value and +the setting to add. Managed sources load into their own `database` and `schema`, then the +target's, as described in [Targets](#targets). A target that names an unknown connection fails when the configuration loads, before anything connects. From d1afa3f1ae4cd3b2c50e673e17965d9838356845 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:21:32 +0000 Subject: [PATCH 08/11] fix: check ignored connection schemas where locations go missing, and keep preserve consistent for loaders --- .../_helpers/compile_driver/targets.rs | 24 +++++++-- .../bindings/_helpers/load/load_selection.rs | 8 +-- .../cli/commands/main/compile/_test_types.py | 5 +- .../compile/test_missing_target_database.py | 53 ++++++++++++++++--- .../compile/_helpers/test_assembly.py | 17 ++++++ 5 files changed, 88 insertions(+), 19 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 708a5372aa..563bcd5300 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -173,6 +173,9 @@ pub(crate) fn named_target_schema_strategy( selected_target: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = discovered.py(); + if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? { + return Ok(()); + } let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; let target_name: Bound<'_, PyAny> = resolve_target_name( @@ -197,9 +200,6 @@ pub(crate) fn named_target_schema_strategy( let connection: Bound<'_, PyAny> = build_effective_connection_config(discovered, Some(target_name), None, false)?; let help: Option = match text_or_none(&mapping_get(&connection, "schema")?)? { - None if native_field(adapter, "allows_implicit_managed_write_schema")?.is_truthy()? => { - return Ok(()); - } Some(value) => { message.push_str(&ignored_connection_note("schema", &value)); Some(project_setting_help( @@ -595,6 +595,24 @@ pub(crate) fn managed_write_schemas( (project_schema.as_deref(), ignored.as_deref()), )?; } + if ignored.is_none() { + return Ok(()); + } + for loader in native_field(project, "loader_functions")?.try_iter()? { + let loader: Bound<'_, PyAny> = loader?; + let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; + if destination.is_some_and(|destination| destination.contains('.')) { + continue; + } + managed_write_schema( + &format!( + "Intermediate loader '{}'", + native_field(&loader, "name")?.str()?.to_cow()? + ), + &loader.py().None().into_bound(loader.py()), + (project_schema.as_deref(), ignored.as_deref()), + )?; + } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 02d076e9db..f3fda6852f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -7,7 +7,6 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_core::load_command::errors::UnknownLoadSelector; use sqlbuild_core::load_command::main::load_reference_loaders::load_reference_loaders; use sqlbuild_core::load_command::main::select_load_entries::select_load_entries as select_entries; @@ -277,14 +276,9 @@ pub(crate) fn loader_defaults<'py>( let defaults: Bound<'py, PyAny> = native_field(&native_field(discovered, "project_config")?, "defaults")?; let location = |field: &str, connection_value: Option>| { - let target: Bound<'py, PyAny> = target_value(target_config, field)?; project_location( ( - if target.eq(PRESERVE_TARGET_VALUE)? { - py.None().into_bound(py) - } else { - target - }, + target_value(target_config, field)?, native_field(&defaults, field)?, ), connection_value.into_iter().filter(|_| inline).collect(), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 45f691e682..230ba9bd54 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1242,6 +1242,7 @@ class MissingTargetDatabaseTestCase: project_toml: str expected_error: str expected_example_count: int + extra_files: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) @@ -1425,8 +1426,8 @@ class PythonMacroArgumentErrorTestCase: @dataclass(frozen=True) -class InlineConnectionLocationTestCase: - """A project whose only location is its inline connection, and where its models build.""" +class ConnectionLocationTestCase: + """A project's connection and defaults, and where its models build.""" description: str project_toml: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index c47390faab..2cc89918b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -10,7 +10,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - InlineConnectionLocationTestCase, + ConnectionLocationTestCase, MissingTargetDatabaseTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -72,10 +72,39 @@ ), expected_example_count=0, ), + MissingTargetDatabaseTestCase( + description="an ignored connection schema stops a bare intermediate loader", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\n' + 'database = "dev.duckdb"\nschema = "raw"\n\n' + '[connections.archive]\ndatabase = "archive.duckdb"\n' + ), + extra_files={ + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized view, " + "schema staging);\n\nSELECT 1 AS order_id\n" + ), + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[S103]: Intermediate loader 'raw_orders' has no explicitly resolved " + "physical write schema. Set schema on the resource or its defaults, selected " + "target, loader_schema, or connection. The connection's schema 'raw' is not used " + "because the project defines named connections or targets.\n" + " = help: to set project-wide defaults, set this in sqlbuild_project.toml:\n" + " [defaults]\n" + ' schema = "raw"\n' + ), + expected_example_count=1, + ), ), ids=lambda case: case.description, ) -def test_given_snowflake_model_without_location_when_compiling_then_error_names_the_setting( +def test_given_model_without_location_when_compiling_then_error_names_the_setting( test_case: MissingTargetDatabaseTestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( @@ -87,7 +116,8 @@ def test_given_snowflake_model_without_location_when_compiling_then_error_names_ "MODEL (description 'Test model orders.', materialized view);\n\n" "SELECT 1 AS order_id\n" ), - }, + } + | test_case.extra_files, ) result: subprocess.CompletedProcess[str] = run_sqb( @@ -102,7 +132,7 @@ def test_given_snowflake_model_without_location_when_compiling_then_error_names_ @pytest.mark.parametrize( "test_case", ( - InlineConnectionLocationTestCase( + ConnectionLocationTestCase( description="an inline connection database and schema place snowflake models", project_toml=( 'name = "orders"\nadapter = "snowflake"\n\n[connection]\naccount = "example"\n' @@ -110,7 +140,7 @@ def test_given_snowflake_model_without_location_when_compiling_then_error_names_ ), expected_qualified_name="ANALYTICS.ORDERS.orders", ), - InlineConnectionLocationTestCase( + ConnectionLocationTestCase( description="an inline connection schema places bigquery models beside defaults", project_toml=( 'name = "orders"\nadapter = "bigquery"\n\n[connection]\n' @@ -119,11 +149,20 @@ def test_given_snowflake_model_without_location_when_compiling_then_error_names_ ), expected_qualified_name="`example-project.orders.orders`", ), + ConnectionLocationTestCase( + description="a duckdb target without a schema builds into the defaults schema", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\npath = "dev.duckdb"\nschema = "raw"\n\n' + '[defaults]\nschema = "analytics"\n\n[targets.dev]\n' + ), + expected_qualified_name="analytics.orders", + ), ), ids=lambda case: case.description, ) -def test_given_inline_connection_location_when_compiling_then_models_build_there( - test_case: InlineConnectionLocationTestCase, tmp_path: Path +def test_given_connection_location_when_compiling_then_models_build_there( + test_case: ConnectionLocationTestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 84f47a13cc..05602f383f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -848,6 +848,23 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph expected_effective_target_database="example-project", expected_effective_target_schema="orders", ), + AssembleCompiledProjectEffectiveTargetTestCase( + description="a preserved target schema reaches models and loaders alike", + project_toml=""" +name = "demo" +adapter = "duckdb" +default_target = "dev" + +[defaults] +schema = "analytics" + +[targets.dev] +schema = "preserve" +""".strip() + + "\n", + expected_effective_target_database=None, + expected_effective_target_schema="preserve", + ), AssembleCompiledProjectEffectiveTargetTestCase( description="a top-level connection beside named targets is not used", project_toml=""" From 4cc8d121c7c478ec576fae67d0607242cc8effe6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:27:37 +0000 Subject: [PATCH 09/11] fix: stop sqb load when a bare loader's connection schema is not used --- .../_helpers/compile_driver/targets.rs | 2 +- .../bindings/_helpers/load/load_command.rs | 7 +++- .../bindings/_helpers/load/load_selection.rs | 37 +++++++++++++++++++ .../cli/commands/main/compile/_test_types.py | 1 + .../compile/test_missing_target_database.py | 32 +++++++++++++++- 5 files changed, 76 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 563bcd5300..baacaa6a95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -617,7 +617,7 @@ pub(crate) fn managed_write_schemas( } /// Raise when neither the resource nor the project names a physical write schema. -fn managed_write_schema( +pub(crate) fn managed_write_schema( resource_label: &str, schema: &Bound<'_, PyAny>, (project_schema, ignored): (Option<&str>, Option<&str>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 3e4284bff7..d82b7fb7be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -33,7 +33,7 @@ use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; use crate::bindings::_helpers::load::load_selection::{ - LoadProject, LoaderDefaults, loader_defaults, + LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, }; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; @@ -118,6 +118,11 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> = project.reference_entries(&selected, &defaults)?; let machine_output: bool = native_field(request, "json_output")?.extract()?; let use_color: bool = !native_field(request, "no_color")?.extract::()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index f3fda6852f..8aa416d540 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -18,6 +18,7 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::compile_attachment::target::{ connection_is_default_target, project_location, }; +use crate::bindings::_helpers::compile_driver::targets::managed_write_schema; use crate::bindings::_helpers::contracts::spec_resolution::loader_destination_parts; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::host; @@ -290,6 +291,42 @@ pub(crate) fn loader_defaults<'py>( }) } +/// Raise when a selected loader has no schema because the connection's schema is not used. +pub(crate) fn loader_write_schemas<'py>( + discovered: &Bound<'py, PyAny>, + (selected_target, cli_vars): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + entries: &[Bound<'py, PyAny>], +) -> PyResult<()> { + if connection_is_default_target(discovered)? { + return Ok(()); + } + let connection: Bound<'py, PyDict> = build_effective_connection_config( + &discovered.clone(), + Some(selected_target.clone()).filter(|value| !value.is_none()), + Some(cli_vars.clone()).filter(|value| !value.is_none()), + false, + )? + .downcast_into()?; + let Some(ignored) = non_empty_text(connection.get_item("schema")?)? else { + return Ok(()); + }; + let ignored: String = ignored.extract()?; + for entry in entries { + if native_field(entry, "loader")?.is_none() { + continue; + } + managed_write_schema( + &format!( + "Intermediate loader '{}'", + native_field(entry, "name")?.str()? + ), + &native_field(entry, "schema")?, + (None, Some(&ignored)), + )?; + } + Ok(()) +} + fn names(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(Vec::new()); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 230ba9bd54..9bcec8782d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1243,6 +1243,7 @@ class MissingTargetDatabaseTestCase: expected_error: str expected_example_count: int extra_files: dict[str, str] = field(default_factory=dict) + command: tuple[str, ...] = ("--no-color", "compile") @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index 2cc89918b1..f5c5ccacd6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -101,6 +101,36 @@ ), expected_example_count=1, ), + MissingTargetDatabaseTestCase( + description="sqb load stops a bare loader whose connection schema is not used", + project_toml=( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\n' + 'database = "dev.duckdb"\nschema = "raw"\n\n' + '[connections.archive]\ndatabase = "archive.duckdb"\n' + ), + extra_files={ + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized view, " + "schema staging);\n\nSELECT 1 AS order_id\n" + ), + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + expected_error=( + "error[S103]: Intermediate loader 'raw_orders' has no explicitly resolved " + "physical write schema. Set schema on the resource or its defaults, selected " + "target, loader_schema, or connection. The connection's schema 'raw' is not used " + "because the project defines named connections or targets.\n" + " = help: to set project-wide defaults, set this in sqlbuild_project.toml:\n" + " [defaults]\n" + ' schema = "raw"\n' + ), + expected_example_count=1, + command=("--no-color", "load", "--select", "raw_orders"), + ), ), ids=lambda case: case.description, ) @@ -121,7 +151,7 @@ def test_given_model_without_location_when_compiling_then_error_names_the_settin ) result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "compile"), project_dir=project_dir + command=test_case.command, project_dir=project_dir ) assert result.returncode == 1, result.stdout + result.stderr From 5746bb3080078ecfaff566a1228288719ceb10f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:56:19 +0000 Subject: [PATCH 10/11] test: give the write-policy case the project schema an inline connection now supplies --- .../pipeline/_helpers/target_validation/_test_types.py | 1 + .../compiler/pipeline/_helpers/target_validation/helpers.py | 6 +++++- .../pipeline/_helpers/target_validation/test_main.py | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py index 30fbe439c9..6943402bca 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py @@ -19,6 +19,7 @@ class ValidateManagedWriteSchemaTestCase: target_schema: str | None effective_connection: dict[str, object] expected_error_fragment: str | None + effective_target_schema: str | None = None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py index 4a5832323b..afba121b9d 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py @@ -35,7 +35,10 @@ def _execute(self, connection: Any, sql: str) -> Any: def build_project( - *, target: CompiledRelationLocation, effective_connection: dict[str, object] + *, + target: CompiledRelationLocation, + effective_connection: dict[str, object], + effective_target_schema: str | None = None, ) -> CompiledProjectView: """Build a minimal compiled project for target validation tests.""" @@ -43,6 +46,7 @@ def build_project( run_id="run_123", effective_target_name=None, effective_connection=effective_connection, + effective_target_schema=effective_target_schema, effective_vars={}, models=( CompiledModel( diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py index c7a0cdfd5e..7dc1c10e46 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py @@ -217,10 +217,11 @@ def test_given_warehouse_managed_write_without_schema_when_validating_then_error expected_error_fragment=None, ), ValidateManagedWriteSchemaTestCase( - description="connection schema satisfies write policy", + description="an inline connection's schema, as the project schema, satisfies write policy", adapter=PostgresAdapter(), target_schema=None, effective_connection={"schema": "analytics"}, + effective_target_schema="analytics", expected_error_fragment=None, ), ValidateManagedWriteSchemaTestCase( @@ -244,6 +245,7 @@ def test_given_explicit_or_permitted_schema_when_validating_managed_write_then_i qualified_name=None, ), effective_connection=test_case.effective_connection, + effective_target_schema=test_case.effective_target_schema, ) validate_managed_write_schemas(adapter=test_case.adapter, project=project) From 5201eddab1ed117197964b42b6e584d3b410bc4b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:32:48 +0000 Subject: [PATCH 11/11] fix: keep managed source loaders out of the intermediate loader schema check A managed source's loader writes to the source's own schema or the target's loader_schema, so an unused connection schema no longer reports it as an intermediate loader without a schema. Two older tests follow the connection location rules: debug reports no database for a lone named connection, and the Snowflake early-failure case sets no database anywhere, since a lone top-level connection now supplies one. --- .../_helpers/compile_driver/targets.rs | 15 ++++- .../cli/commands/main/compile/_test_types.py | 9 +++ .../compile/test_missing_target_database.py | 55 +++++++++++++++++++ .../commands/main/test_default_connection.py | 2 +- .../sqlbuild/compiler/pipeline/test_main.py | 5 +- 5 files changed, 80 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index baacaa6a95..bba27d6475 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -1,6 +1,6 @@ //! Adapter-aware target namespace validation of a compile. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -598,10 +598,21 @@ pub(crate) fn managed_write_schemas( if ignored.is_none() { return Ok(()); } + let mut source_loaders: HashSet = HashSet::new(); + for source in native_field(project, "sources")?.try_iter()? { + if let Some(loader) = text_or_none(&native_field( + &native_field(&source?, "source_entry")?, + "loader", + )?)? { + source_loaders.insert(loader); + } + } for loader in native_field(project, "loader_functions")?.try_iter()? { let loader: Bound<'_, PyAny> = loader?; let destination: Option = text_or_none(&native_field(&loader, "destination")?)?; - if destination.is_some_and(|destination| destination.contains('.')) { + if destination.is_some_and(|destination| destination.contains('.')) + || source_loaders.contains(&native_field(&loader, "name")?.extract::()?) + { continue; } managed_write_schema( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 9bcec8782d..a742d1f25e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1433,3 +1433,12 @@ class ConnectionLocationTestCase: description: str project_toml: str expected_qualified_name: str + + +@dataclass(frozen=True) +class ManagedSourceLoaderLocationTestCase: + """A command that runs a managed source's loader and the rows its loader schema must hold.""" + + description: str + command: tuple[str, ...] + expected_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py index f5c5ccacd6..088defa40c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_missing_target_database.py @@ -7,10 +7,12 @@ from pathlib import Path from typing import Any +import duckdb import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( ConnectionLocationTestCase, + ManagedSourceLoaderLocationTestCase, MissingTargetDatabaseTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -215,3 +217,56 @@ def test_given_connection_location_when_compiling_then_models_build_there( assert [model["qualified_name"] for model in payload["resources"]["models"]] == [ test_case.expected_qualified_name ], test_case.description + + +@pytest.mark.parametrize( + "test_case", + ( + ManagedSourceLoaderLocationTestCase( + description="build loads the managed source into the target loader schema", + command=("--no-color", "build"), + expected_rows=((1,),), + ), + ), + ids=lambda case: case.description, +) +def test_given_ignored_connection_schema_when_building_managed_source_then_loader_schema_is_used( + test_case: ManagedSourceLoaderLocationTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[connection]\n' + 'database = "dev.duckdb"\nschema = "landing"\n\n' + '[connections.archive]\ndatabase = "archive.duckdb"\n\n' + '[targets.dev]\nloader_schema = "raw"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized view, " + 'schema staging);\n\nSELECT order_id FROM __source("raw_orders")\n' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_orders(ctx: object) -> list[dict[str, object]]:\n" + ' """Raw orders."""\n _ = ctx\n return [{"order_id": 1}]\n' + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + with duckdb.connect(str(project_dir / "dev.duckdb"), read_only=True) as connection: + rows: list[tuple[object, ...]] = connection.execute( + "SELECT order_id FROM raw.raw_orders" + ).fetchall() + assert tuple(rows) == test_case.expected_rows diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py b/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py index cb7ab33bc4..726428d9cc 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py @@ -156,7 +156,7 @@ def test_given_local_and_cli_overrides_when_building_then_precedence_picks_the_r project_toml=_ONE_CONNECTION_PROJECT, argv=("debug", "--no-connection"), expected_exit_code=0, - expected_output_fragments=("connection: local", "shop.duckdb"), + expected_output_fragments=("connection: local", "database: not configured"), ), DefaultConnectionCliTestCase( description="build fails at config load among several named connections", diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 727a3a2a26..351e5001f9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -541,17 +541,16 @@ def test_given_progress_callback_when_building_project_graph_then_reports_compil "test_case", [ SnowflakeTargetValidationIntegrationTestCase( - description="snowflake compile requires explicit target database and schema", + description="snowflake compile without a database anywhere fails early", project_files={ "sqlbuild_project.toml": ( - 'name = "demo"\nadapter = "duckdb"\n\n[connection]\ndatabase = "demo.duckdb"\n' + 'name = "demo"\nadapter = "duckdb"\n\n[connection]\npath = "demo.duckdb"\n' ), "sqlbuild_local.toml": ( 'adapter = "snowflake"\n\n' "[connection]\n" 'account = "${ENV:TEST_ACCOUNT}"\n' 'warehouse = "TEST_WH"\n' - 'database = "TEST_DB"\n' 'schema = "TEST_SCHEMA"\n' ), "models/stg_orders.sql": "MODEL (description 'Test model stg_orders.', materialized view);\n\nSELECT 1 AS order_id\n",