diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index c91ac3996b..8d090ec92f 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -185,6 +185,7 @@ jobs: RELEASE_PR: ${{ needs.release-please.outputs.release_pr }} REQUEST_ID: ${{ github.run_id }}-${{ github.run_attempt }} run: | + set -uo pipefail head_sha="$(git rev-parse HEAD)" recovery="rerun the failed Release Please job to re-check" post_status() { @@ -231,15 +232,18 @@ jobs: workflow="${workflows[${index}]}" context="${contexts[${index}]}" run_id="" - for _ in {1..30}; do + # A dispatched run can take minutes to appear in the run list, and a failed lookup + # is retried rather than read as "not started yet". + for _ in {1..60}; do run_id="$(gh run list --workflow "${workflow}" --branch "${RELEASE_BRANCH}" \ --event workflow_dispatch --limit 50 \ --json databaseId,headSha,displayTitle,createdAt \ - --jq ".[] | select(.headSha == \"${head_sha}\" + --jq "[.[] | select(.headSha == \"${head_sha}\" and (.displayTitle | endswith(\" [${REQUEST_ID}]\")) - and .createdAt >= \"${dispatched_after}\") | .databaseId" | head -n 1)" + and .createdAt >= \"${dispatched_after}\") | .databaseId] | first // empty")" \ + || run_id="" [[ -n "${run_id}" ]] && break - sleep 2 + sleep 10 done if [[ -z "${run_id}" ]]; then post_status failure "${context}" "Trusted release check did not start" diff --git a/Cargo.lock b/Cargo.lock index 56a195c5bf..b76db7f99d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1408,9 +1408,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "fensu-facts" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaaeb4fa29af8fa378ebd826e8a28655dfd13ed7acb98919dbe64f882681fef" +checksum = "8ca6517dd7fb6fd5ec99d7c25be8ca860e4d753afb081cc0ab6a065aac40d0a8" dependencies = [ "dunce", "get-size2", @@ -1426,9 +1426,9 @@ dependencies = [ [[package]] name = "fensu-native" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8053936f5ea33f32a301649d2178fad69cee1fe8aff7f17b35d56ba88d85024b" +checksum = "dbba81e35d514d8c889ef10287202e6c7920bdf43094ceb925616fded7cd5b92" dependencies = [ "fensu-facts", "globset", diff --git a/Cargo.toml b/Cargo.toml index f6fc2c295a..6dadb24d1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,8 +83,8 @@ tower-http = { version = "0.6", default-features = false, features = ["compressi fensu-policy = "=0.22.0" # Fensu's Python facts and core rules without their Python bindings: the default `python` # feature adds PyO3, rusqlite, flate2 and sha2. -fensu-facts = { version = "=0.31.0", default-features = false } -fensu-native = { version = "=0.31.0", default-features = false } +fensu-facts = { version = "=0.31.1", default-features = false } +fensu-native = { version = "=0.31.1", default-features = false } futures-util = { version = "0.3", default-features = false, features = ["std"] } # `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. getrandom = "0.4" diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 728798a5cb..0bd47022ec 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -124,8 +124,8 @@ project and does not change it: `edge_running_peaks`, that a pre-hook of `int_edge_basket_running_peaks` creates (`models/intermediate/edge_cases/_sqlbuild/_hooks/sql/`) and SQLBuild does not declare. -It has 167 models (13 staging, 140 intermediate, 14 marts), 5 Python functions, a SQL table -function, a hook-created JavaScript function, 192 unit tests and 4 scenarios, in nine groups: +It has 168 models (13 staging, 141 intermediate, 14 marts), 5 Python functions, a SQL table +function, a hook-created JavaScript function, 193 unit tests and 4 scenarios, in nine groups: | Group | Covers | |-------|--------| @@ -136,7 +136,7 @@ function, a hook-created JavaScript function, 192 unit tests and 4 scenarios, in | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes, and a scoped SQL macro collapsing whitespace whose body opens with a block comment and holds a line comment; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | | Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts; `LAG` and `LEAD` with one, two and three arguments; a comma-joined `LATERAL` whose nested derived table reads the left item; relations aliased `first` and `last`, one without `AS`, the words of Snowflake's read-order clauses; untyped and typed `FILTER` lambdas over `ARRAY_AGG` in a grouped query; column expression ranges in compiled SQL that SQL and Python macros, a variable and cursor bounds rewrote, and columns qualified by their reference call, whose ranges are dropped | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | -| Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild, and a keyed cursor `delete_insert` copy of it whose rows move out of the window between loads | +| Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild, a keyed cursor `delete_insert` copy of it whose rows move out of the window between loads, and a cursor `delete_insert` copy of every version that `replace_key` replaces per shipment | | Translation refusals | one model per construct that translation tools refuse or guard, most putting a construct refused only under strict checking before one refused by default, so a model shows both: `FLOAT` values through sums, averages, running totals, arrays, objects, `HASH`, `TO_CHAR` and `::NUMBER(38, 10)`; `VARIANT` dates, decimals and XML; `ARRAY_AGG` without `WITHIN GROUP`, sparse arrays and `ARRAY(NUMBER)` casts; `REDUCE`, `FILTER` and `TRANSFORM` lambdas; patterns and `REGEXP_REPLACE` replacements built from columns, and lazy patterns; `TRY_TO_DATE` with a format from a column; zoned literals, `CONVERT_TIMEZONE` from `CURRENT_TIMESTAMP`, `SEQ8`, `GENERATOR` with an expression `ROWCOUNT`; `PIVOT`, `ANY_VALUE`, `IGNORE NULLS` around `IFF`, unaliased columns read by Snowflake's name, `UNION` branches whose column is always NULL; `TO_NUMBER`, `TO_DECIMAL` and `TRY_BASE64_DECODE_BINARY`; an open (uncontracted) source `edge__supplier_notes`, the table function, read for one basket and after `LATERAL` for every basket, and the hook-created function. `rpt_edge_case_model_keys` and `rpt_edge_case_array_model_keys` read every model in the group, so a refusal there also refuses a downstream model | Unit-test expectations in the edge-case area are captured on Snowflake under the `edge` target's diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_versions_replaced.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_versions_replaced.sql new file mode 100644 index 0000000000..824b95b9c0 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_delivery_versions_replaced.sql @@ -0,0 +1,53 @@ +MODEL ( + description """ + Every delivered version of each shipment with a version loaded in the window, replaced as a + group by delete and insert. A shipment has several rows, so replace_key names the group key: + unique_key would promise one row per shipment. + """, + tags [edge_case], + materialized incremental, + incremental_strategy delete_insert, + replace_key [shipment_id], + cursor source_loaded_at, + cursor_type timestamp, + cursor_grain second, + cursor_inputs ( + edge__delivery_updates _loaded_at, + ), + columns ( + delivery_id (type VARCHAR, nullable false, audits [not_null, unique]), + shipment_id (type VARCHAR, nullable false, audits [not_null]), + delivery_status (type VARCHAR), + is_delete (type BOOLEAN), + version_seq (type NUMBER(38, 0)), + source_loaded_at (type TIMESTAMP_NTZ, nullable false, audits [not_null]), + ), +); + +WITH touched AS ( + SELECT DISTINCT u.shipment_id + FROM __source("edge__delivery_updates") AS u + WHERE + u._loaded_at >= __cursor_start() + AND u._loaded_at < __cursor_end() +), + +final AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.delivery_status, + v.is_delete, + v.version_seq, + MAX(v.loaded_at) OVER (PARTITION BY v.shipment_id) AS source_loaded_at + FROM __ref("int_edge_delivery_update_versions") AS v + WHERE + v.loaded_at < __cursor_end() + AND v.shipment_id IN ( + SELECT t.shipment_id + FROM touched AS t + ) +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_versions_replaced.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_versions_replaced.sql new file mode 100644 index 0000000000..0b48869d43 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_delivery_versions_replaced.sql @@ -0,0 +1,83 @@ +TEST ( + name "int_edge_delivery_versions_replaced__copies_every_version_of_shipments_touched_in_the_window", + cursor_start "2026-03-02 00:00:00", + cursor_end "2026-03-03 00:00:00", +); + +WITH __source__edge__delivery_updates AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v._op, + v._seq::NUMBER(38, 0) AS _seq, + v._loaded_at::TIMESTAMP_NTZ AS _loaded_at + FROM VALUES + ('DU-01', 'SH-1', 'DS-O-3001', 'label_created', '2026-02-27', NULL, 'I', 10, '2026-03-01 10:00:00'), + ('DU-04', 'SH-3', 'DS-O-3003', 'delivered', '2026-02-20', '2026-02-25', 'I', 30, '2026-03-01 10:00:00'), + ('DU-06', 'SH-1', 'DS-O-3001', 'picked_up', '2026-02-27', NULL, 'U', 11, '2026-03-02 10:00:00'), + ('DU-08', 'SH-2', 'DS-O-3002', 'delivered', '2026-02-28', '2026-03-02', 'U', 21, '2026-03-02 23:59:59'), + ('DU-11', 'SH-6', 'DS-O-3006', 'label_created', '2026-03-02', NULL, 'I', 60, '2026-03-03 00:00:00') AS v(delivery_id, shipment_id, order_ref, delivery_status, ordered_on, delivered_on, _op, _seq, _loaded_at) +), + +__ref__int_edge_delivery_update_versions AS ( + SELECT + v.delivery_id, + v.shipment_id, + v.order_ref, + v.delivery_status, + v.ordered_on::DATE AS ordered_on, + v.delivered_on::DATE AS delivered_on, + v.is_delete, + v.version_seq::NUMBER(38, 0) AS version_seq, + v.loaded_at::TIMESTAMP_NTZ AS loaded_at + FROM VALUES + ('DU-01', 'SH-1', 'DS-O-3001', 'label_created', '2026-02-27', NULL, FALSE, 10, '2026-03-01 10:00:00'), + ('DU-02', 'SH-1', 'DS-O-3001', 'in_transit', '2026-02-27', NULL, FALSE, 12, '2026-03-01 10:00:00'), + ('DU-04', 'SH-3', 'DS-O-3003', 'delivered', '2026-02-20', '2026-02-25', FALSE, 30, '2026-03-01 10:00:00'), + ('DU-06', 'SH-1', 'DS-O-3001', 'picked_up', '2026-02-27', NULL, FALSE, 11, '2026-03-02 10:00:00'), + ('DU-08', 'SH-2', 'DS-O-3002', 'delivered', '2026-02-28', '2026-03-02', FALSE, 21, '2026-03-02 23:59:59'), + ('DU-12', 'SH-1', 'DS-O-3001', 'delivered', '2026-02-27', '2026-03-03', FALSE, 13, '2026-03-03 00:00:00'), + ('DU-11', 'SH-6', 'DS-O-3006', 'label_created', '2026-03-02', NULL, FALSE, 60, '2026-03-03 00:00:00') AS v( + delivery_id, + shipment_id, + order_ref, + delivery_status, + ordered_on, + delivered_on, + is_delete, + version_seq, + loaded_at + ) +), + +__expected__int_edge_delivery_versions_replaced AS ( + SELECT + 'DU-01' AS delivery_id, + 'SH-1' AS shipment_id, + 10 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'DU-02' AS delivery_id, + 'SH-1' AS shipment_id, + 12 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'DU-06' AS delivery_id, + 'SH-1' AS shipment_id, + 11 AS version_seq, + '2026-03-02 10:00:00'::TIMESTAMP_NTZ(9) AS source_loaded_at + UNION ALL + SELECT + 'DU-08' AS delivery_id, + 'SH-2' AS shipment_id, + 21 AS version_seq, + '2026-03-02 23:59:59'::TIMESTAMP_NTZ(9) AS source_loaded_at +) + +SELECT 1 diff --git a/crates/sqlbuild-analysis/src/planner_identity/constants.rs b/crates/sqlbuild-analysis/src/planner_identity/constants.rs index 23d471a561..79f8e591ff 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/constants.rs @@ -1,7 +1,7 @@ //! Planner identity constants shared with the Python planner. /// Model config keys whose values participate in version identity, in Python's insertion order after `cursor_inputs`. -pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 39] = [ +pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 40] = [ "append_cursor_inclusive", "batch_size", "check_columns", @@ -33,6 +33,7 @@ pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 39] = [ "snapshot_schema_change", "snapshot_strategy", "unique_key", + "replace_key", "updated_at", "valid_from_column", "valid_to_column", diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs index 1176dc46a1..93e559e8b7 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -86,6 +86,8 @@ pub const PERMANENT_TABLE_TYPE: &str = "permanent"; pub const TRANSIENT_TABLE_DDL: &str = "TRANSIENT TABLE"; /// The permanent table DDL replacing the transient keyword. pub const PERMANENT_TABLE_DDL: &str = "TABLE"; +/// SQL dialects whose `delete_insert` cannot replace several rows per key. +pub const GROUP_REPLACEMENT_UNSUPPORTED_DIALECTS: [&str; 1] = ["bigquery"]; /// The warning code of an incremental model whose table keeps an outdated physical layout. pub const PHYSICAL_LAYOUT_PENDING_WARNING_CODE: &str = "PHYSICAL_LAYOUT_PENDING"; /// Model config keys of a table's physical layout, in the order warnings name them. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/group_replacement_refusal.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/group_replacement_refusal.rs new file mode 100644 index 0000000000..c58d1b9c5f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/group_replacement_refusal.rs @@ -0,0 +1,20 @@ +//! The refusal of `replace_key` on adapters that cannot replace groups of rows. + +use crate::planner_strategy::constants::GROUP_REPLACEMENT_UNSUPPORTED_DIALECTS; + +/// Why `model_name` cannot set `replace_key` on an adapter with SQL dialect `dialect`, if it cannot. +pub fn group_replacement_refusal( + model_name: &str, + (adapter_name, dialect): (&str, Option<&str>), + has_replace_key: bool, +) -> Option { + let unsupported: bool = has_replace_key + && dialect.is_some_and(|dialect| GROUP_REPLACEMENT_UNSUPPORTED_DIALECTS.contains(&dialect)); + unsupported.then(|| { + format!( + "model '{model_name}' sets replace_key, but adapter '{adapter_name}' cannot replace \ + groups of rows; use unique_key with one row per key, or replace groups on any \ + adapter other than BigQuery" + ) + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs index 7d2258959c..c590fc0f0b 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs @@ -7,6 +7,7 @@ pub mod count_timeline_batches; pub mod cursor_input_column; pub mod cursor_type_warning; pub mod effective_run_scope; +pub mod group_replacement_refusal; pub mod lineage_reference; pub mod materialization_type; pub mod microbatch_limit_warning; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs index 80d6b312c9..b7de0e65bc 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_group_replacement_refusal.rs"] +mod group_replacement_refusal; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_model_entry.rs"] diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_group_replacement_refusal.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_group_replacement_refusal.rs new file mode 100644 index 0000000000..da0b2f5bb5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_group_replacement_refusal.rs @@ -0,0 +1,56 @@ +use crate::planner_strategy::main::group_replacement_refusal::group_replacement_refusal; +use crate::planner_strategy::tests::test_types::GroupReplacementRefusalTestCase; + +#[test] +fn given_replace_key_when_planning_then_only_bigquery_refuses_it() { + let test_cases = [ + GroupReplacementRefusalTestCase { + description: "BigQuery's delete_insert matches one source row per target row", + dialect: Some("bigquery"), + has_replace_key: true, + expected_refused: true, + }, + GroupReplacementRefusalTestCase { + description: "BigQuery with unique_key keeps its delete_insert", + dialect: Some("bigquery"), + has_replace_key: false, + expected_refused: false, + }, + GroupReplacementRefusalTestCase { + description: "Snowflake deletes every row of a key", + dialect: Some("snowflake"), + has_replace_key: true, + expected_refused: false, + }, + GroupReplacementRefusalTestCase { + description: "an adapter without SQL analysis", + dialect: None, + has_replace_key: true, + expected_refused: false, + }, + ]; + + for test_case in test_cases { + let refusal = group_replacement_refusal( + "forecast_lines", + ("warehouse", test_case.dialect), + test_case.has_replace_key, + ); + + assert_eq!( + refusal.is_some(), + test_case.expected_refused, + "{}", + test_case.description + ); + } + assert_eq!( + group_replacement_refusal("forecast_lines", ("bigquery", Some("bigquery")), true) + .as_deref(), + Some( + "model 'forecast_lines' sets replace_key, but adapter 'bigquery' cannot replace groups of \ + rows; use unique_key with one row per key, or replace groups on any adapter other \ + than BigQuery" + ) + ); +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs index 611b3fa01e..3f33a4b74f 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_types.rs @@ -104,6 +104,13 @@ pub(super) struct TableTypeDdlTestCase { pub(super) expected_ddl: &'static str, } +pub(super) struct GroupReplacementRefusalTestCase { + pub(super) description: &'static str, + pub(super) dialect: Option<&'static str>, + pub(super) has_replace_key: bool, + pub(super) expected_refused: bool, +} + pub(super) struct PhysicalLayoutWarningTestCase { pub(super) description: &'static str, pub(super) materialization: MaterializationType, diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 8c7ec909fa..722e754a54 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -104,7 +104,7 @@ pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "unaccounted_partition_policy", ]; /// The MODEL header keys, which `[path_defaults]` entries may set. -pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ +pub(crate) const MODEL_HEADER_KEYS: [&str; 74] = [ "alias", "append_cursor_inclusive", "audit_factories", @@ -158,6 +158,7 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "post_hooks", "pre_hooks", "primary_key", + "replace_key", "replay_on_change", "row_diff_exclude_columns", "row_diff_sample_rows", diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 86a7a4dcd5..f6c8b8b807 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -54,7 +54,7 @@ pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits"; /// The singular audit directory below an audit role. pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular"; /// Header keys a `MODEL` statement accepts, sorted by code point. -pub const SQL_MODEL_HEADER_KEYS: [&str; 74] = [ +pub const SQL_MODEL_HEADER_KEYS: [&str; 75] = [ "alias", "append_cursor_inclusive", "audit_factories", @@ -109,6 +109,7 @@ pub const SQL_MODEL_HEADER_KEYS: [&str; 74] = [ "post_hooks", "pre_hooks", "primary_key", + "replace_key", "replay_on_change", "row_diff_exclude_columns", "row_diff_sample_rows", diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs index 28c2055977..599e3ac240 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use crate::model_validation::_helpers::config::{ - ConfigView, has_config_value, one_of, sorted_values, string_sequence, + ConfigView, has_config_value, non_blank_string_check, one_of, sorted_values, string_sequence, }; use crate::model_validation::_helpers::config::{model_header_help, python_lower}; use crate::model_validation::_helpers::cursor_bounds::{ @@ -14,9 +14,9 @@ use crate::model_validation::constants::{ APPEND_STRATEGY, BOUNDED_REPLAY_PREFIX, CURSOR_GRAIN_BATCH_SIZES, CURSOR_INPUT_KEYS, CURSOR_POLICY_DISABLED, CURSOR_TYPES, DELETE_INSERT_STRATEGY, EFFECTIVE_BATCH_SIZE, ENFORCED_CONTRACT, FUTURE_CURSOR_ACTIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_STRATEGIES, - MERGE_STRATEGY, ON_SCHEMA_CHANGE_EXAMPLE, ON_SCHEMA_CHANGE_POLICIES, REPLAY_ON_CHANGE_ACTIONS, - REPLAY_ON_CHANGE_EXAMPLE, REPLAY_ON_CHANGE_VALID_VALUES, TABLE_MATERIALIZATION, - TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, + MERGE_STRATEGY, MICROBATCH_MODE, ON_SCHEMA_CHANGE_EXAMPLE, ON_SCHEMA_CHANGE_POLICIES, + REPLAY_ON_CHANGE_ACTIONS, REPLAY_ON_CHANGE_EXAMPLE, REPLAY_ON_CHANGE_VALID_VALUES, + TABLE_MATERIALIZATION, TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, }; use crate::model_validation::models::{ModelValidationFacts, ValidationStop}; use crate::model_validation::types::Check; @@ -43,6 +43,7 @@ pub(crate) struct IncrementalValues { pub(crate) cursor_start: Option, pub(crate) cursor_end: Option, pub(crate) unique_key: Option, + pub(crate) replace_key: Option, pub(crate) cursor_inputs: Option, pub(crate) max_microbatches: Option, pub(crate) microbatch_limit: Option, @@ -79,6 +80,7 @@ impl IncrementalValues { cursor_start: node("cursor_start"), cursor_end: node("cursor_end"), unique_key: node("unique_key"), + replace_key: node("replace_key"), cursor_inputs: node("cursor_inputs"), max_microbatches: node("max_microbatches"), microbatch_limit: node("microbatch_limit"), @@ -368,8 +370,36 @@ fn check_write_strategy( ) -> Check { let strategy = values.strategy.as_deref(); let has_unique_key = has_config_value(values.unique_key.as_ref()); - if strategy == Some(DELETE_INSERT_STRATEGY) && values.cursor.is_none() && !has_unique_key { - return Err(config.error("delete_insert without cursor requires unique_key")); + if let Some(replace_key) = &values.replace_key { + let valid: bool = if replace_key.kind() == NodeKind::Str { + non_blank_string_check(replace_key) + } else { + is_non_empty_string_list(replace_key) && !replace_key.items().is_empty() + }; + if !valid { + return Err(config.error("replace_key must be a column name or a list of column names")); + } + if strategy != Some(DELETE_INSERT_STRATEGY) { + return Err(config.error("replace_key requires incremental_strategy=delete_insert")); + } + if values.incremental_mode.as_deref() == Some(MICROBATCH_MODE) { + return Err(config.error( + "replace_key is not supported with incremental_mode=microbatch; each batch only replaces its own window", + )); + } + if has_unique_key { + return Err(config.error( + "set either unique_key (one row per key) or replace_key (groups of rows), not both", + )); + } + } + let has_replace_key = values.replace_key.is_some(); + if strategy == Some(DELETE_INSERT_STRATEGY) + && values.cursor.is_none() + && !has_unique_key + && !has_replace_key + { + return Err(config.error("delete_insert without cursor requires unique_key or replace_key")); } if strategy == Some(MERGE_STRATEGY) && !has_unique_key { return Err(config.error("merge strategy requires unique_key")); @@ -435,6 +465,12 @@ fn check_contract_columns( &string_sequence(values.unique_key.as_ref()), &declared, )?; + require_declared( + config, + "replace_key", + &string_sequence(values.replace_key.as_ref()), + &declared, + )?; require_declared( config, "merge_exclude_columns", diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index 74d56179a8..87b9298bc3 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -121,12 +121,13 @@ pub const SQL_WILDCARD: &str = "*"; pub const PLACEHOLDER_SIGIL: &str = "@@@"; /// Keys that only incremental models may set. -pub const INCREMENTAL_ONLY_KEYS: [&str; 5] = [ +pub const INCREMENTAL_ONLY_KEYS: [&str; 6] = [ "on_schema_change", "replay_on_change", "append_cursor_inclusive", "merge_exclude_columns", "full_refresh", + "replace_key", ]; /// Cursor input keys that require a cursor-based incremental model. pub const CURSOR_INPUT_KEYS: [&str; 3] = [ diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs index 6cdfdea1b5..24ecccd910 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs @@ -337,6 +337,101 @@ fn given_effective_configs_when_validating_natively_then_python_outcomes_result( query_sql: "select 1", expected_outcome: "model 'orders_daily': merge_exclude_columns cannot include unique_key column(s): order_id", }, + ValidateTestCase { + description: "delete_insert replacing groups of rows by replace_key", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("delete_insert")), + ("replace_key", strings(&["forecast_version"])), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "delete_insert without a cursor or a key", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("delete_insert")), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': delete_insert without cursor requires unique_key or replace_key", + }, + ValidateTestCase { + description: "replace_key beside unique_key", + config: incremental(vec![ + ("incremental_strategy", Value::Str("delete_insert")), + ("unique_key", Value::Str("order_id")), + ("replace_key", Value::Str("forecast_version")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': set either unique_key (one row per key) or replace_key (groups of rows), not both", + }, + ValidateTestCase { + description: "replace_key on a merge", + config: incremental(vec![ + ("incremental_strategy", Value::Str("merge")), + ("unique_key", Value::Str("order_id")), + ("replace_key", Value::Str("forecast_version")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': replace_key requires incremental_strategy=delete_insert", + }, + ValidateTestCase { + description: "replace_key on a microbatch", + config: incremental(vec![ + ("incremental_strategy", Value::Str("delete_insert")), + ("incremental_mode", Value::Str("microbatch")), + ("replace_key", Value::Str("forecast_version")), + ("microbatch_strategy", Value::Str("watermark")), + ("cursor_watermark_mode", Value::Str("all")), + ("batch_size", Value::Str("1d")), + ("lookback", Value::Str("3d")), + ( + "cursor_inputs", + map(vec![( + "orders", + map(vec![ + ("column", Value::Str("updated_at")), + ("roles", strings(&["filter", "watermark"])), + ]), + )]), + ), + ( + "microbatch_limit", + map(vec![ + ("max_batches", Value::Int(5)), + ("action", Value::Str("cap_from_start")), + ]), + ), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': replace_key is not supported with incremental_mode=microbatch; each batch only replaces its own window", + }, + ValidateTestCase { + description: "replace_key on a table", + config: vec![ + ("materialized", Value::Str("table")), + ("replace_key", Value::Str("forecast_version")), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': replace_key is only valid for incremental models", + }, + ValidateTestCase { + description: "an empty replace_key", + config: incremental(vec![ + ("incremental_strategy", Value::Str("delete_insert")), + ("replace_key", strings(&[])), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': replace_key must be a column name or a list of column names", + }, ValidateTestCase { description: "a bounded replay policy", config: incremental(vec![("replay_on_change", Value::Str("bounded- 14d"))]), diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs index 789ab2d276..7f60864225 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/evaluation.rs @@ -1,10 +1,11 @@ //! The Python template evaluator: references, `if`, `eq`, `ne` and `coalesce`. use crate::templates::_helpers::missing_values::is_missing_value_message; +use crate::templates::_helpers::substitution::rendered; use crate::templates::errors::TemplateError; use crate::templates::main::template_error_message::template_error_message; use crate::templates::models::{ - ContextValue, Expression, Scalar, TemplateFailure, TemplateOptions, + ContextValue, Expression, Scalar, TemplateFailure, TemplateOptions, TextPart, }; use crate::templates::types::TemplateHost; @@ -30,6 +31,18 @@ pub(crate) fn evaluate( ) -> Result { match expression { Expression::Text(text) => host.text(text), + Expression::Interpolated(parts) => { + let mut joined = String::new(); + for part in parts { + match part { + TextPart::Text(text) => joined.push_str(text), + TextPart::Template { body, expression } => { + joined.push_str(&rendered(host, body, expression, options)?); + } + } + } + host.text(&joined) + } Expression::Reference(reference) => evaluate_reference(host, reference, options), Expression::Function { name, arguments } => { evaluate_function(host, name, arguments, options) diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs b/crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs index f61c2df1f2..014cacb0ba 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/expressions.rs @@ -49,6 +49,10 @@ impl Parser { self.index += 1; Ok(Expression::Text(text)) } + TokenKind::Interpolated { parts, .. } => { + self.index += 1; + Ok(Expression::Interpolated(parts)) + } TokenKind::Word(word) => { self.index += 1; if self.match_symbol('(') { diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs b/crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs index 8c21b517fd..ff4d7a1fef 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/pattern.rs @@ -1,27 +1,96 @@ -//! Python's `\$\{([^{}]+)\}` matches, found left to right without overlap. +//! Where each `${...}` template in a string starts and ends. -/// Return the byte spans of every template match, including the `${` and `}`. +const TEMPLATE_OPEN: &str = "${"; +const ESCAPE: char = '\\'; +/// How deep templates may nest inside quoted arguments; a deeper one does not close. +const MAX_NESTING: usize = 8; + +/// Byte spans of each template, `${` to `}`, closing outside quotes or else at the first brace. pub(crate) fn template_spans(text: &str) -> Vec<(usize, usize)> { - let bytes = text.as_bytes(); + counted_template_spans(text).0 +} + +/// `template_spans`, and how many characters the scan visited. +pub(crate) fn counted_template_spans(text: &str) -> (Vec<(usize, usize)>, usize) { let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut steps: usize = 0; let mut index = 0; - while index + 1 < bytes.len() { - if bytes[index] == b'$' && bytes[index + 1] == b'{' { - let body = index + 2; - let close = bytes[body..] - .iter() - .position(|byte| matches!(byte, b'{' | b'}')) - .map(|offset| body + offset); - if let Some(close) = close - && close > body - && bytes[close] == b'}' - { - spans.push((index, close + 1)); - index = close + 1; - continue; + while let Some(offset) = text[index..].find(TEMPLATE_OPEN) { + let start = index + offset; + let body = start + TEMPLATE_OPEN.len(); + let (quoted, quoted_steps) = nested_body_length(&text[body..], 0); + steps += quoted_steps; + let length = match quoted { + Some(length) => Some(length), + None => { + let (first_brace, brace_steps) = brace_free_length(&text[body..]); + steps += brace_steps; + first_brace } + }; + if let Some(length) = length { + spans.push((start, body + length + 1)); + index = body + length + 1; + } else { + index = start + 1; } - index += 1; } - spans + (spans, steps) +} + +/// Byte length of the body at the start of `body` up to its `}`, skipping quoted arguments. +pub(crate) fn template_body_length(body: &str) -> Option { + nested_body_length(body, 0).0 +} + +/// The quoted-argument-aware body length, if it closes, and the characters visited. +fn nested_body_length(body: &str, depth: usize) -> (Option, usize) { + if depth > MAX_NESTING { + return (None, 0); + } + let mut steps: usize = 0; + let mut quote: Option = None; + let mut index = 0; + while let Some(character) = body[index..].chars().next() { + steps += 1; + let mut next = index + character.len_utf8(); + match quote { + Some(_) if character == ESCAPE => { + next += body[next..].chars().next().map_or(0, char::len_utf8); + } + Some(open) if character == open => quote = None, + Some(_) if body[index..].starts_with(TEMPLATE_OPEN) => { + let nested = index + TEMPLATE_OPEN.len(); + let (length, nested_steps) = nested_body_length(&body[nested..], depth + 1); + steps += nested_steps; + let Some(length) = length else { + return (None, steps); + }; + next = nested + length + 1; + } + Some(_) => {} + None if is_quote(character) => quote = Some(character), + None if character == '{' => return (None, steps), + None if character == '}' => return ((index > 0).then_some(index), steps), + None => {} + } + index = next; + } + (None, steps) +} + +/// Python's rule, the body up to its first brace if that is a non-empty body's `}`, and the steps. +fn brace_free_length(body: &str) -> (Option, usize) { + let Some(close) = body.find(['{', '}']) else { + return (None, body.len()); + }; + ( + (close > 0 && body[close..].starts_with('}')).then_some(close), + close + 1, + ) +} + +/// Whether `character` opens a quoted argument. +pub(crate) fn is_quote(character: char) -> bool { + matches!(character, '\'' | '"') } diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs b/crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs index c562ed4191..415122358a 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/substitution.rs @@ -2,7 +2,7 @@ use crate::templates::_helpers::evaluation::evaluate; use crate::templates::_helpers::expressions::parse_expression; -use crate::templates::models::{TemplateFailure, TemplateOptions}; +use crate::templates::models::{Expression, TemplateFailure, TemplateOptions}; use crate::templates::types::TemplateHost; /// `text` with every template at `spans` replaced by its value rendered as text. @@ -17,12 +17,20 @@ pub(crate) fn substituted( for &(start, end) in spans { expanded.push_str(&text[copied..start]); let body = &text[start + 2..end - 1]; - let expression = parse_expression(body)?; - let value = evaluate(host, &expression, options)?; - let label = format!("{} variable '{body}'", host.label()); - expanded.push_str(&host.render(&value, &label)?); + expanded.push_str(&rendered(host, body, &parse_expression(body)?, options)?); copied = end; } expanded.push_str(&text[copied..]); Ok(expanded) } + +/// The value of the template `body`, parsed as `expression`, rendered as text. +pub(crate) fn rendered( + host: &H, + body: &str, + expression: &Expression, + options: TemplateOptions, +) -> Result { + let value = evaluate(host, expression, options)?; + host.render(&value, &format!("{} variable '{body}'", host.label())) +} diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs index 636b459877..30a6d40716 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/tokens.rs @@ -1,12 +1,19 @@ //! The Python template tokenizer. +use crate::templates::_helpers::expressions::parse_expression; +use crate::templates::_helpers::pattern::{is_quote, template_body_length}; use crate::templates::errors::TemplateError; -use crate::templates::models::TemplateFailure; +use crate::templates::models::{TemplateFailure, TextPart}; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum TokenKind { Word(String), Text(String), + /// A quoted argument holding `${...}` templates, and its text as written. + Interpolated { + written: String, + parts: Vec, + }, Symbol(char), End, } @@ -22,7 +29,9 @@ impl Token { /// Return the token's value as Python's error messages show it. pub(crate) fn value(&self) -> String { match &self.kind { - TokenKind::Word(text) | TokenKind::Text(text) => text.clone(), + TokenKind::Word(text) + | TokenKind::Text(text) + | TokenKind::Interpolated { written: text, .. } => text.clone(), TokenKind::Symbol(symbol) => symbol.to_string(), TokenKind::End => String::new(), } @@ -35,6 +44,15 @@ const ESCAPE: char = '\\'; /// Split one template expression into tokens. pub(crate) fn tokenize(expression: &str) -> Result, TemplateFailure> { let characters: Vec = expression.chars().collect(); + let text = QuotedSource { + expression, + offsets: expression + .char_indices() + .map(|(offset, _)| offset) + .chain([expression.len()]) + .collect(), + characters: &characters, + }; let mut tokens: Vec = Vec::new(); let mut index = 0; while let Some(&character) = characters.get(index) { @@ -47,9 +65,9 @@ pub(crate) fn tokenize(expression: &str) -> Result, TemplateFailure> }); index += 1; } else if is_quote(character) { - let (text, next) = quoted_text(&characters, index)?; + let (kind, next) = quoted_text(&text, index)?; tokens.push(Token { - kind: TokenKind::Text(text), + kind, position: index, }); index = next; @@ -74,9 +92,11 @@ pub(crate) fn tokenize(expression: &str) -> Result, TemplateFailure> Ok(tokens) } -fn quoted_text(characters: &[char], start: usize) -> Result<(String, usize), TemplateFailure> { +fn quoted_text(text: &QuotedSource, start: usize) -> Result<(TokenKind, usize), TemplateFailure> { + let characters: &[char] = text.characters; let quote = characters[start]; - let mut text = String::new(); + let mut parts: Vec = Vec::new(); + let mut literal = String::new(); let mut index = start + 1; while let Some(&character) = characters.get(index) { if character == ESCAPE { @@ -85,12 +105,28 @@ fn quoted_text(characters: &[char], start: usize) -> Result<(String, usize), Tem TemplateError::UnterminatedEscape { position: index }, )); }; - text.push(escaped); + literal.push(escaped); index += 2; } else if character == quote { - return Ok((text, index + 1)); + let kind = if parts.is_empty() { + TokenKind::Text(literal) + } else { + parts.push(TextPart::Text(literal)); + TokenKind::Interpolated { + written: characters[start + 1..index].iter().collect(), + parts, + } + }; + return Ok((kind, index + 1)); + } else if let Some((body, length)) = nested_template(text, index) { + parts.push(TextPart::Text(std::mem::take(&mut literal))); + parts.push(TextPart::Template { + expression: parse_expression(&body)?, + body, + }); + index += length; } else { - text.push(character); + literal.push(character); index += 1; } } @@ -103,6 +139,20 @@ fn quoted_text(characters: &[char], start: usize) -> Result<(String, usize), Tem )) } -fn is_quote(character: char) -> bool { - matches!(character, '\'' | '"') +/// The body of a complete `${...}` template at character `index`, and its length in characters. +fn nested_template(text: &QuotedSource, index: usize) -> Option<(String, usize)> { + let characters: &[char] = text.characters; + if characters.get(index) != Some(&'$') || characters.get(index + 1) != Some(&'{') { + return None; + } + let rest: &str = &text.expression[text.offsets[index + 2]..]; + let body: &str = &rest[..template_body_length(rest)?]; + Some((body.to_owned(), body.chars().count() + 3)) +} + +/// One expression as characters, with each character's byte offset and the length at the end. +struct QuotedSource<'t> { + expression: &'t str, + offsets: Vec, + characters: &'t [char], } diff --git a/crates/sqlbuild-model-config/src/templates/models.rs b/crates/sqlbuild-model-config/src/templates/models.rs index f1d25bea65..28763911b3 100644 --- a/crates/sqlbuild-model-config/src/templates/models.rs +++ b/crates/sqlbuild-model-config/src/templates/models.rs @@ -6,6 +6,8 @@ use crate::templates::errors::TemplateError; #[derive(Clone, Debug, PartialEq, Eq)] pub enum Expression { Text(String), + /// A quoted argument holding `${...}` templates, joined as text. + Interpolated(Vec), Reference(String), Function { name: String, @@ -13,6 +15,16 @@ pub enum Expression { }, } +/// One piece of a quoted argument: literal text, or a template rendered as text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TextPart { + Text(String), + Template { + body: String, + expression: Expression, + }, +} + /// A value as the Python resolver compares and tests it: `None`, a `bool` or its `str()`. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Scalar { diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs index 81a7a8c31c..af876fb3ca 100644 --- a/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs +++ b/crates/sqlbuild-model-config/src/templates/tests/test_expand_template_string.rs @@ -1,10 +1,13 @@ use std::cell::RefCell; +use crate::templates::_helpers::pattern::counted_template_spans; use crate::templates::errors::TemplateError; use crate::templates::main::expand_template_string::expand_template_string; use crate::templates::models::{StringExpansion, TemplateFailure, TemplateOptions}; use crate::templates::tests::helpers::{text, value}; -use crate::templates::tests::test_types::{TemplateTestCase, TestHost, Value}; +use crate::templates::tests::test_types::{ + TemplateScanTestCase, TemplateTestCase, TestHost, Value, +}; #[test] fn given_template_strings_when_expanding_then_values_and_reads_match_python() { @@ -190,6 +193,75 @@ fn given_template_strings_when_expanding_then_values_and_reads_match_python() { expected: value(Value::Text("prod".to_owned())), expected_reads: &[], }, + TemplateTestCase { + description: "a quoted argument interpolates templates, so coalesce builds a fallback", + text: "${coalesce(ENV:MISSING, 'dev_${ENV:SCHEMA_NAME}')}", + expected: value(Value::Text("dev_analytics".to_owned())), + expected_reads: &["ENV:MISSING", "ENV:SCHEMA_NAME"], + }, + TemplateTestCase { + description: "a set first argument wins before the quoted fallback is read", + text: "${coalesce(ENV:SCHEMA_NAME, 'dev_${ENV:MISSING}')}", + expected: value(Value::Text("analytics".to_owned())), + expected_reads: &["ENV:SCHEMA_NAME"], + }, + TemplateTestCase { + description: "a missing variable inside the last quoted argument names itself", + text: "${coalesce(ENV:MISSING, 'dev_${ENV:UNSET}')}", + expected: Err(TemplateFailure::Missing(TemplateError::MissingEnvironment( + "UNSET".to_owned(), + ))), + expected_reads: &["ENV:MISSING", "ENV:UNSET", "ENV:UNSET"], + }, + TemplateTestCase { + description: "nested templates in if arguments may reuse the outer quote", + text: "${if(eq(env, 'prod'), 'p_${coalesce(ENV:MISSING, 'x')}', 'n')}", + expected: value(Value::Text("p_x".to_owned())), + expected_reads: &["ENV:MISSING"], + }, + TemplateTestCase { + description: "an interpolated argument inside text renders into it", + text: "schema_${coalesce(ENV:EMPTY, 'dev_${env}_${flag}')}_x", + expected: text("schema_dev_prod_true_x"), + expected_reads: &["ENV:EMPTY"], + }, + TemplateTestCase { + description: "a backslash keeps a literal template in a quoted argument", + text: r"${coalesce('\${ENV:SCHEMA_NAME}')}", + expected: value(Value::Text("${ENV:SCHEMA_NAME}".to_owned())), + expected_reads: &[], + }, + TemplateTestCase { + description: "an unclosed template in a quoted argument keeps the outer one from closing", + text: "${coalesce('a${b')}", + expected: Err(TemplateFailure::Invalid( + TemplateError::UnterminatedString { + quote: "single", + position: 1, + }, + )), + expected_reads: &[], + }, + TemplateTestCase { + description: "templates nested more than eight deep do not close", + text: "${coalesce('${coalesce('${coalesce('${coalesce('${coalesce('${coalesce('${coalesce('${coalesce('${coalesce('${env}')}')}')}')}')}')}')}')}')}", + expected: text("${coalesce('prod')}"), + expected_reads: &[], + }, + TemplateTestCase { + description: "a brace inside a quoted argument is text", + text: "${coalesce('a}b')}", + expected: value(Value::Text("a}b".to_owned())), + expected_reads: &[], + }, + TemplateTestCase { + description: "an object cannot be interpolated into a quoted argument", + text: "${coalesce('x_${opaque}')}", + expected: Err(TemplateFailure::Invalid(TemplateError::Message( + "model config variable 'opaque' is an object".to_owned(), + ))), + expected_reads: &[], + }, ]; for test_case in test_cases { @@ -224,3 +296,33 @@ fn given_template_strings_when_expanding_then_values_and_reads_match_python() { ); } } + +#[test] +fn given_malformed_nested_templates_when_finding_templates_then_the_scan_stays_linear() { + let test_cases: [TemplateScanTestCase; 3] = [ + TemplateScanTestCase { + description: "an unclosed chain of nested templates with alternating quotes", + text: format!("${{'{}", "${\"${'".repeat(4_000)), + expected_max_steps_per_character: 20, + }, + TemplateScanTestCase { + description: "many unclosed templates that each open a quote", + text: "${'a ".repeat(20_000), + expected_max_steps_per_character: 20, + }, + TemplateScanTestCase { + description: "many sibling templates inside one quoted argument", + text: format!("${{coalesce('{}')}}", "${env}".repeat(20_000)), + expected_max_steps_per_character: 20, + }, + ]; + for test_case in test_cases { + let (_, steps) = counted_template_spans(&test_case.text); + assert!( + steps <= test_case.expected_max_steps_per_character * test_case.text.len(), + "{}: {steps} steps over {} characters", + test_case.description, + test_case.text.len() + ); + } +} diff --git a/crates/sqlbuild-model-config/src/templates/tests/test_types.rs b/crates/sqlbuild-model-config/src/templates/tests/test_types.rs index 18cdac2a02..32e7d6d811 100644 --- a/crates/sqlbuild-model-config/src/templates/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/templates/tests/test_types.rs @@ -95,3 +95,10 @@ pub(super) struct TemplateTestCase { pub(super) expected: Result, TemplateFailure>, pub(super) expected_reads: &'static [&'static str], } + +/// A malformed template string and the most characters per input character its scan may visit. +pub(super) struct TemplateScanTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) expected_max_steps_per_character: usize, +} diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 76c6261f51..c849c68bd0 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -619,6 +619,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("start_cursor_config", FieldDefault::None), FieldSpec::new("invocation_time", FieldDefault::None), FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("replace_key", FieldDefault::EmptyTuple), FieldSpec::new("merge_exclude_columns", FieldDefault::EmptyTuple), FieldSpec::new("snapshot_strategy", FieldDefault::None), FieldSpec::new("updated_at_column", FieldDefault::None), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index eff32c12da..61a80e0b5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -25,5 +25,7 @@ pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; pub(crate) const PHYSICAL_STORAGE_DIALECT: &str = "clickhouse"; /// The error code for physical storage keys on an adapter without them. pub(crate) const PHYSICAL_STORAGE_UNSUPPORTED_CODE: &str = "A011"; +/// The error code for `replace_key` on an adapter that cannot replace groups of rows. +pub(crate) const GROUP_REPLACEMENT_UNSUPPORTED_CODE: &str = "A012"; /// Plan action values that create the model's table anew or leave it untouched. pub(crate) const TABLE_RECREATING_ACTIONS: [&str; 2] = ["create_table", "skip"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 20efc7a4c6..39ea87e0f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_analysis::planner_strategy::main::group_replacement_refusal::group_replacement_refusal; use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_lookback; use sqlbuild_analysis::planner_strategy::main::physical_layout_warning::physical_layout_warning; @@ -19,8 +20,9 @@ use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::resource_reads; use crate::bindings::_helpers::planner_strategy::constants::{ - CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, - PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, TABLE_RECREATING_ACTIONS, + CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, GROUP_REPLACEMENT_UNSUPPORTED_CODE, + MICROBATCH_MODE, PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, + TABLE_RECREATING_ACTIONS, }; use crate::bindings::_helpers::planner_strategy::cursor_types::{ cursor_type_warning_record, warehouse_column_types, @@ -662,6 +664,26 @@ pub(crate) fn assemble_model_plan_entry<'py>( (&enforcement, &warehouse_columns), )?; let unique_key: Bound<'py, PyTuple> = string_list(&values.0, "unique_key")?; + let replace_key: Bound<'py, PyTuple> = string_list(&values.0, "replace_key")?; + if let Some(refusal) = group_replacement_refusal( + &model_reads::model_name(model)?, + ( + &native_field(&request.adapter, "adapter_name")? + .str()? + .to_string(), + native_field(&request.adapter, "sql_analysis_dialect_name")? + .extract::>()? + .as_deref(), + ), + !replace_key.is_empty()?, + ) { + return Err(input_error( + py, + ErrorId::CompileInput, + &refusal, + Some(GROUP_REPLACEMENT_UNSUPPORTED_CODE), + )); + } let merge_exclude: Bound<'py, PyTuple> = config_text_tuple(py, &values, "merge_exclude_columns")? .unwrap_or_else(|| PyTuple::empty(py)); @@ -672,9 +694,14 @@ pub(crate) fn assemble_model_plan_entry<'py>( )?, "value", )?; + let delete_key: &Bound<'py, PyTuple> = if replace_key.is_empty()? { + &unique_key + } else { + &replace_key + }; let ddl: String = logical_ddl( request, - (&unique_key, &merge_exclude), + (delete_key, &merge_exclude), (&warehouse_columns, ddl_cursor_bounds(request)?), )?; let entry: Bound<'py, PyDict> = PyDict::new(py); @@ -691,6 +718,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( config_fields(request, &values, &entry)?; execution_limit_fields(model, &values, &entry)?; entry.set_item("unique_key", unique_key)?; + entry.set_item("replace_key", replace_key)?; entry.set_item("merge_exclude_columns", merge_exclude)?; entry.set_item("table_type", table_type)?; storage_fields(request, &values, &entry)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index cbbae49101..ae576d3f91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -35,8 +35,9 @@ use sqlbuild_python_front::model_config::types::PyConfigNode; /// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; -const METADATA_CONFIG_KEYS: [&str; 4] = [ +const METADATA_CONFIG_KEYS: [&str; 5] = [ "unique_key", + "replace_key", "cursor", "partition_column", "row_diff_exclude_columns", diff --git a/crates/sqlbuild-refactor/src/refactoring/constants.rs b/crates/sqlbuild-refactor/src/refactoring/constants.rs index 8f7ebeb913..f209691ed6 100644 --- a/crates/sqlbuild-refactor/src/refactoring/constants.rs +++ b/crates/sqlbuild-refactor/src/refactoring/constants.rs @@ -29,9 +29,10 @@ pub const GENERIC_AUDIT_DIRECTORY: &str = "generic"; pub const RELATIONSHIPS_AUDIT: &str = "relationships"; pub const RELATIONSHIPS_TO_KEY: &str = "to"; pub const RELATIONSHIPS_FIELD_KEY: &str = "field"; -pub const COLUMN_VALUED_CONFIG_KEYS: [&str; 9] = [ +pub const COLUMN_VALUED_CONFIG_KEYS: [&str; 10] = [ "cursor", "unique_key", + "replace_key", "updated_at", "check_columns", "merge_exclude_columns", diff --git a/fensu.toml b/fensu.toml index 8e8fb2c0f1..f6d3b5122e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1676,8 +1676,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 169863 } -reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." +thresholds = { max_crate_code_lines = 169893 } +reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out. Raised from 169863: replace_key plan-entry plumbing and the A012 refusal, +31 with sqlbuild-python-base." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] diff --git a/pyproject.toml b/pyproject.toml index 424515febe..59a9686774 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.31.0", + "fensu>=0.31.1", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index a0ea0cf0da..0653c5dd08 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -64,7 +64,8 @@ table more than once. ### delete_insert -Deletes the rows the new delta replaces, then inserts the delta. Requires either `cursor` or `unique_key`. +Deletes the rows the new delta replaces, then inserts the delta. Requires `cursor`, `unique_key` or +`replace_key`. ```sql MODEL ( @@ -85,13 +86,35 @@ Which rows it deletes depends on the model's settings: | Settings | Rows deleted before the insert | |----------|--------------------------------| | `cursor` | Rows whose cursor falls in the replay window | -| `unique_key` | Rows whose key appears in the delta | -| `cursor` and `unique_key` | Rows in the replay window, and rows whose key appears in the delta | +| `unique_key` | Rows whose key appears in the delta. The key must hold one row per value; for groups of rows per value, use `replace_key` | +| `replace_key` | Rows whose `replace_key` values appear in the delta | +| `cursor` and `unique_key` or `replace_key` | Rows in the replay window, and rows whose key appears in the delta | | `incremental_mode microbatch` | Rows in the batch's window; a `unique_key` only matches rows in diff and review | With both a cursor and a key, a row whose cursor value moved out of the window, such as an order updated after its first load, replaces its earlier version instead of adding a second row. +`unique_key` promises one row per key. Use `replace_key` when a key value covers a group of rows +that each load replaces whole, such as the lines of one forecast version: + +```sql +MODEL ( + description "Forecast lines, several per forecast version", + materialized incremental, + incremental_strategy delete_insert, + replace_key [forecast_version], +); + +SELECT forecast_version, product_id, forecast_month, quantity +FROM __ref("stg_forecast_lines") +``` + +A delta holding version `2026-06` deletes every `2026-06` row and inserts the new ones. Other +versions keep their rows. A model sets `unique_key` or `replace_key`, not both, and `replace_key` +is valid only with `delete_insert` and not with `incremental_mode microbatch`. BigQuery rejects +`replace_key` with `A012` when planning; every other adapter replaces groups. Rules that need a +proven key, such as `SQLCLEAN018`, use `unique_key` and never `replace_key`. + ### merge Upserts rows using a unique key. Matched rows are updated; unmatched rows are inserted. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md index 6b54363cfe..72b6c8528e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/interpolation.md @@ -11,6 +11,7 @@ Online: https://sqlbuild.com/docs/concepts/interpolation/ - Syntax reference - Project variables - Environment variables +- Config value functions - Context variables - Deferred placeholders - Audit parameters @@ -39,6 +40,7 @@ The rule is simple: if it's any SQL that will be executed, it uses `@`. If it's | `@name` / `@'name'` | Generic audit SQL | Raw / SQL-literal audit argument | | `${CTX:...}` | TOML/YAML config values | Config compilation | | `${ENV:...}` | TOML/YAML config values | Config compilation | +| `${coalesce(...)}`, `${if(...)}`, `${eq(...)}`, `${ne(...)}` | TOML/YAML config values | Config compilation - see [Config value functions](#config-value-functions) | `@@CTX:` is intentionally SQL-hook-only. Model SQL describes a relation's data and should not reference its own destination identity. SQL hooks are the operational SQL layer where destination context is useful - grants, logging, post-materialization DDL. Python hooks access the same information through `ctx.destination` on the [`HookContext`](models/hooks/python.md#hook-context) object. @@ -106,6 +108,38 @@ WHERE source_system = '@@ENV:SOURCE_SYSTEM' If the environment variable is not set, SQLBuild raises a compile error. +## Config value functions + +A `${...}` config value can choose between values with a function. This target uses +`WAREHOUSE_SCHEMA` when it is set, and builds `DEV_` from `WAREHOUSE_USER` otherwise: + +```toml +[targets.dev] +schema = "${coalesce(ENV:WAREHOUSE_SCHEMA, 'DEV_${ENV:WAREHOUSE_USER}')}" +``` + +| Function | Returns | +|----------|---------| +| `coalesce(a, b, ...)` | The first argument that is set and true; otherwise the last argument | +| `if(condition, a, b)` | `a` when `condition` is true, otherwise `b` | +| `eq(a, b)` | `true` when `a` and `b` read the same as text | +| `ne(a, b)` | `true` when `a` and `b` read differently as text | + +An argument is a project variable (`schema_prefix`), `ENV:NAME`, `CTX:name`, `true`, `false`, +`null`, a quoted string or another function call. A quoted string can hold `${...}` templates, +which render into it as text, as in `'DEV_${ENV:WAREHOUSE_USER}'`. + +Details: + +- An argument is false when it is `null`, empty or blank, `0`, or `false` in any letter case. +- A missing variable makes its argument missing, so `coalesce` moves on to the next argument. When + every argument is missing, SQLBuild reports the missing variable in the last one. +- A backslash in a quoted string keeps the next character as written, so `'\${ENV:USER}'` is the + text `${ENV:USER}`. Inside a TOML or YAML double-quoted value, write the backslash twice: + `"${coalesce('\\${ENV:USER}')}"`. +- A template inside a quoted string can use the same quotes as the string around it. Templates nest + up to eight levels deep; a template nested deeper stays as written text. + ## Context variables Context variables provide access to the current model's destination relation, active target, and run metadata. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index f4f1fb3ed2..b0a06f4675 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -248,6 +248,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged | `cursor_future_action` | `cap` or `error` when a watermark exceeds `cursor_future_max_distance` | | `cursor_inputs` | Upstream names mapped to cursor columns | | `unique_key` | Row identity. `merge` and `delete_insert` replace rows with a matching key; `append` and microbatch `delete_insert` use it only to match rows in diff and review | +| `replace_key` | `delete_insert` only: replaces every row whose `replace_key` values appear in the delta. Use it instead of `unique_key` when one key value covers several rows | | `incremental_mode` | Set to `microbatch` for batched execution | | `microbatch_strategy` | Required in microbatch mode: `watermark` or `rolling_window` | | `cursor_watermark_mode` | Watermark strategy policy: `all` or `any` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 79ad15afca..66aa68cf3c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -238,8 +238,8 @@ max_ranking_order_by = 5 # default Keys come from `unique` column audits and model-level `unique` audits naming a column, when the column is also non-null (a `not_null` audit or `nullable false`), and from the `unique_key` of incremental `merge` models and of `delete_insert` models matched by key without a cursor. - Snapshot, append, microbatch and cursor-ranged `delete_insert` `unique_key`s do not prove one - row per key. Keys are followed through projections, renames, filters, one-to-one joins, + Snapshot, append, microbatch and cursor-ranged `delete_insert` `unique_key`s, and every + `replace_key`, do not prove one row per key. Keys are followed through projections, renames, filters, one-to-one joins, `GROUP BY` grains (named or positional), `SELECT DISTINCT`, `UNION`, and `ROW_NUMBER() ... = 1` deduplication in `QUALIFY` or a later `WHERE`; a `QUALIFY` key holds for later steps, never for the window that defines it. A join that can match a row more than once, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 180d5af6a7..25edbba368 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -105,7 +105,7 @@ dlt_sources: | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | | `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | | `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | -| `primary_key` / `merge_key` | dlt keys for `merge`. | +| `primary_key` / `merge_key` | dlt keys for `merge`. A `merge_key` replaces every row whose key value arrives, like a SQL model's `delete_insert` with [`replace_key`](../concepts/incremental.md#delete_insert). | | `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | | `schema` | Per-resource schema override (falls back to the group `schema`). | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md index 60544e7351..8fd6812a78 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md @@ -102,7 +102,7 @@ The `ingestr` block on a source supports the following fields: | `replace` | Drop and recreate the destination table (default when no strategy is set) | | `append` | Insert all rows without deduplication | | `merge` | Upsert based on `primary_key`, using `incremental_key` for change detection | -| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements | +| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements. A SQL model does the same with `delete_insert` and a `cursor`; use [`replace_key`](../concepts/incremental.md#delete_insert) to replace groups of rows by key value instead | | `truncate+insert` | Truncate destination table, then insert all rows | ## Template variables diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9f0fe7bc80..d6d38207f2 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13980,6 +13980,7 @@ class ModelPlanEntry(NativeRecord): start_cursor_config: StartCursorsConfig | None invocation_time: datetime | None unique_key: tuple[str, ...] + replace_key: tuple[str, ...] merge_exclude_columns: tuple[str, ...] snapshot_strategy: str | None updated_at_column: str | None @@ -14068,6 +14069,7 @@ class ModelPlanEntry(NativeRecord): start_cursor_config: StartCursorsConfig | None = ..., invocation_time: datetime | None = ..., unique_key: tuple[str, ...] = ..., + replace_key: tuple[str, ...] = ..., merge_exclude_columns: tuple[str, ...] = ..., snapshot_strategy: str | None = ..., updated_at_column: str | None = ..., diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 3a73f620b9..01b9fa6f56 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -902,7 +902,7 @@ def _execute_dml( """Execute the incremental DML strategy from delta into target.""" strategy: str | None = entry.incremental_strategy - unique_key: tuple[str, ...] = entry.unique_key + unique_key: tuple[str, ...] = entry.replace_key or entry.unique_key target_col_set: frozenset[str] = frozenset(column_name_key(col.name) for col in target_columns) intersection_names: tuple[str, ...] = tuple( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py index f1d336126f..9190a6d1c5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py @@ -17,119 +17,132 @@ table_exists, ) +_REPO_FILES: dict[str, str] = { + "sqlbuild_project.toml": dedent( + """ -@pytest.mark.parametrize( - "test_case", - [ - TemplateExpressionsBuildE2ETestCase( - description="build uses if eq ne and coalesce in config interpolation", - repo_files={ - "sqlbuild_project.toml": dedent( - """ + name = "template_expressions_project" - name = "template_expressions_project" + adapter = "duckdb" - adapter = "duckdb" + default_target = "dev" - default_target = "dev" + [connection] - [connection] + database = "template_expressions.duckdb" - database = "template_expressions.duckdb" + [defaults] - [defaults] + materialized = "incremental" - materialized = "incremental" + incremental_strategy = "append" - incremental_strategy = "append" + append_cursor_inclusive = "${if(eq(ENV:APPEND_INCLUSIVE, '0'), false, true)}" - append_cursor_inclusive = "${if(eq(ENV:APPEND_INCLUSIVE, '0'), false, true)}" + [targets] - [targets] + [targets.dev] - [targets.dev] + schema = "${coalesce(ENV:TEMPLATE_SCHEMA, 'tpl_${ENV:TEMPLATE_USER}')}" - schema = "${coalesce(ENV:TEMPLATE_SCHEMA, 'main')}" + """ + ).strip() + + "\n", + "sources/raw.yml": dedent( + """ + sources: + - name: raw_orders + description: Test source raw_orders. + schema: main + table: raw_orders + """ + ).strip() + + "\n", + "models/orders.sql": dedent( + """ + MODEL (description "Test model orders.", + cursor ordered_at, + cursor_type timestamp, + cursor_grain second, + alias '${if(eq(ENV:ALIAS_MODE, "zero"), "orders_zero", if( + ne(ENV:ALIAS_MODE, "zero"), + "orders_nonzero", + "orders_nonzero" + ))}', + ); - """ - ).strip() - + "\n", - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - description: Test source raw_orders. - schema: main - table: raw_orders - """ - ).strip() - + "\n", - "models/orders.sql": dedent( - """ - MODEL (description "Test model orders.", - cursor ordered_at, - cursor_type timestamp, - cursor_grain second, - alias '${if(eq(ENV:ALIAS_MODE, "zero"), "orders_zero", if( - ne(ENV:ALIAS_MODE, "zero"), - "orders_nonzero", - "orders_nonzero" - ))}', - ); + SELECT id, ordered_at + FROM __source("raw_orders") + """ + ).strip() + + "\n", +} +_INITIAL_SEED_SQL: str = dedent( + """ + CREATE TABLE main.raw_orders (id INTEGER, ordered_at TIMESTAMP); + + INSERT INTO main.raw_orders VALUES + (1, '2026-01-01 00:00:00'), + (2, '2026-01-01 01:00:00'); + """ +).strip() +_MUTATION_SQL: tuple[str, ...] = ( + "INSERT INTO main.raw_orders VALUES (3, '2026-01-01 01:00:00')", + "INSERT INTO main.raw_orders VALUES (4, '2026-01-01 02:00:00')", +) +_EXPECTED_RUNTIME_SQL_FRAGMENT: str = ( + "WHERE ordered_at > TIMESTAMP '2026-01-01 00:00:00' " + "AND ordered_at < TIMESTAMP '2026-01-01T02:00:01'" +) +_EXPECTED_QUERY_RESULTS: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] = ( + ( + ( + "SELECT id, CAST(ordered_at AS VARCHAR) " + "FROM tpl_demo.orders_nonzero ORDER BY id, ordered_at" + ), + ( + (2, "2026-01-01 01:00:00"), + (2, "2026-01-01 01:00:00"), + (3, "2026-01-01 01:00:00"), + (4, "2026-01-01 02:00:00"), + ), + ), +) - SELECT id, ordered_at - FROM __source("raw_orders") - """ - ).strip() - + "\n", - }, - initial_seed_sql=dedent( - """ - CREATE TABLE main.raw_orders (id INTEGER, ordered_at TIMESTAMP); - INSERT INTO main.raw_orders VALUES - (1, '2026-01-01 00:00:00'), - (2, '2026-01-01 01:00:00'); - """ - ).strip(), - mutation_sql=( - "INSERT INTO main.raw_orders VALUES (3, '2026-01-01 01:00:00')", - "INSERT INTO main.raw_orders VALUES (4, '2026-01-01 02:00:00')", - ), - env={ - "APPEND_INCLUSIVE": "0", - "TEMPLATE_SCHEMA": "tpl_demo", - "ALIAS_MODE": "one", - }, +@pytest.mark.parametrize( + "test_case", + [ + TemplateExpressionsBuildE2ETestCase( + description="build uses if eq ne, coalesce and quoted-argument templates in config", + repo_files=_REPO_FILES, + initial_seed_sql=_INITIAL_SEED_SQL, + mutation_sql=_MUTATION_SQL, + env={"APPEND_INCLUSIVE": "0", "TEMPLATE_USER": "demo", "ALIAS_MODE": "one"}, + command=("--no-color", "build"), + expected_exit_code=0, + expected_runtime_sql_fragment=_EXPECTED_RUNTIME_SQL_FRAGMENT, + expected_query_results=_EXPECTED_QUERY_RESULTS, + ), + TemplateExpressionsBuildE2ETestCase( + description="a set schema variable wins without reading the quoted fallback's variable", + repo_files=_REPO_FILES, + initial_seed_sql=_INITIAL_SEED_SQL, + mutation_sql=_MUTATION_SQL, + env={"APPEND_INCLUSIVE": "0", "TEMPLATE_SCHEMA": "tpl_demo", "ALIAS_MODE": "one"}, command=("--no-color", "build"), expected_exit_code=0, - expected_runtime_sql_fragment=( - "WHERE ordered_at > TIMESTAMP '2026-01-01 00:00:00' " - "AND ordered_at < TIMESTAMP '2026-01-01T02:00:01'" - ), - expected_query_results=( - ( - ( - "SELECT id, CAST(ordered_at AS VARCHAR) " - "FROM tpl_demo.orders_nonzero ORDER BY id, ordered_at" - ), - ( - (2, "2026-01-01 01:00:00"), - (2, "2026-01-01 01:00:00"), - (3, "2026-01-01 01:00:00"), - (4, "2026-01-01 02:00:00"), - ), - ), - ), - ) + expected_runtime_sql_fragment=_EXPECTED_RUNTIME_SQL_FRAGMENT, + expected_query_results=_EXPECTED_QUERY_RESULTS, + ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py index 9199ee3e4b..7c40868fbb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py @@ -222,6 +222,15 @@ def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_a query_sql=_RENAMED_CHAIN_SQL, expected_codes=(), ), + RankingKeyProofCase( + description="a delete_insert replace_key names groups of rows, not a key", + upstream_header=_INCREMENTAL_HEADER_TEMPLATE.format( + strategy="delete_insert", cursor="" + ).replace("unique_key order_id", "replace_key order_id"), + upstream_sql=_INCREMENTAL_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=("SQLCLEAN018",), + ), RankingKeyProofCase( description="a cursor-ranged delete_insert unique_key is not a key", upstream_header=_INCREMENTAL_HEADER_TEMPLATE.format( diff --git a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py index a7789e5e24..62c6dbaa2f 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py @@ -140,6 +140,48 @@ ), ), ), + BuildExecutionTestCase( + description="delete_insert by replace_key replaces every row of a delta's key values", + project_files={ + "sqlbuild_project.toml": _PROJECT_YML, + "models/forecasts.sql": ( + "MODEL (description 'Forecast lines, several per version.',\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " replace_key [forecast_version],\n" + ");\n\n" + 'SELECT forecast_version, line_no, amount FROM __source("raw_forecasts")' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_forecasts\n description: Test source raw_forecasts.\n" + " schema: main\n table: raw_forecasts\n" + ), + }, + setup_sql=( + "CREATE TABLE main.forecasts AS SELECT * FROM (VALUES " + "('v1', 1, 10), ('v1', 2, 20), ('v2', 1, 30), ('v2', 2, 40)) " + "AS t(forecast_version, line_no, amount)", + "CREATE TABLE raw_forecasts AS SELECT * FROM (VALUES " + "('v2', 1, 31), ('v2', 2, 41), ('v2', 3, 51)) " + "AS t(forecast_version, line_no, amount)", + ), + expected_status=BuildStatus.SUCCESS, + expected_success_count=1, + expected_model_statuses=(("forecasts", ExecutionStatus.SUCCESS),), + expected_query_results=( + ( + "SELECT forecast_version, line_no, amount FROM main.forecasts " + "ORDER BY forecast_version, line_no", + ( + ("v1", 1, 10), + ("v1", 2, 20), + ("v2", 1, 31), + ("v2", 2, 41), + ("v2", 3, 51), + ), + ), + ), + ), BuildExecutionTestCase( description="sql udf builds before dependent model", project_files={ diff --git a/uv.lock b/uv.lock index a8c7cb20c1..8206188894 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.31.0" +version = "0.31.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6b/99/a1692065a637e8d85acef4de5496ca0ba93913f3365920995f4e9290f6eb/fensu-0.31.0.tar.gz", hash = "sha256:af5aad8b7595b521940e548ad11a3c649914c60d7a72e5c16f2486ae0ad00727", size = 537385, upload-time = "2026-11-15T12:31:37.016Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/b8/87deea124f5c859930cb02b3379e0ba9607c0cbf14e3ca5b22b71d7a2b41/fensu-0.31.1.tar.gz", hash = "sha256:fd0eb63cfed3102da0d6af1d97e9c016511e03b0caf1658421649920f099db06", size = 537417, upload-time = "2026-11-16T07:47:26.817Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/de/91648b56387827750097b61e17e395164a42c9aa07a9729df83021d4dec7/fensu-0.31.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:41b87150cd8fc57539c75209e6212e49c047a9b39ed9808aefdd0d61c85b4491", size = 4157866, upload-time = "2026-11-15T12:31:17.872Z" }, - { url = "https://files.pythonhosted.org/packages/74/e7/7d56f99c2edaf220b6669b2b05f4db816ac7b6b8190b5e91c4497448480d/fensu-0.31.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:3b466cbdcdf118a8d5e89f69a38f2c9b17115cb6de3b988443b50b8c6da7b41d", size = 4012595, upload-time = "2026-11-15T12:31:19.625Z" }, - { url = "https://files.pythonhosted.org/packages/e5/78/510278b4542e714cf7c7dd43189ded49de9b8fc7c759cb6c901f1b6e4286/fensu-0.31.0-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:229b8c459a85aa5560ac1b15363df69dd87bc44b68d87147a78ab8ea897fd44b", size = 4249463, upload-time = "2026-11-15T12:31:21.301Z" }, - { url = "https://files.pythonhosted.org/packages/93/d2/eb491ade5cd0b8b40376951965b873688ce05d4aa7d8bfa663ce87c810ea/fensu-0.31.0-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3eed44bc6a830384815dc80e3e9b05ab189c28d49c89a0927b1e88945c01206a", size = 4322607, upload-time = "2026-11-15T12:31:22.777Z" }, - { url = "https://files.pythonhosted.org/packages/bd/44/00d555425c9e9c0b6817416d1bd1f44387e416045c6c94c9793b87bf91ea/fensu-0.31.0-cp312-abi3-win_amd64.whl", hash = "sha256:37e17e24a25f206334f7e26041a378bff9562808aa13ef2a043b0fcee82359b9", size = 3987846, upload-time = "2026-11-15T12:31:24.29Z" }, + { url = "https://files.pythonhosted.org/packages/9a/7e/8052557459d25046afae7993c3849cdc69822c839f8c1bc55c4b6103a380/fensu-0.31.1-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d9237d6c955dc8513f484e1009c0911a47a9cf005668f98ddc51608747173a6f", size = 4154951, upload-time = "2026-11-16T07:47:07.139Z" }, + { url = "https://files.pythonhosted.org/packages/94/9b/30b16901d9129c464267c85f295fa6cc2a3833cab1f32586b23c21d24f3e/fensu-0.31.1-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:a496dcd01e1e0716d6cf358685f009f038d8fd32774946b54dd9ecc714c93e2f", size = 4011426, upload-time = "2026-11-16T07:47:09.385Z" }, + { url = "https://files.pythonhosted.org/packages/e1/52/dc404677eea7fecb421a23d5ed531cd65ad3545a3f4b068ccb99a5ff8a50/fensu-0.31.1-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e56a687b3f87270dfcda7d9f643f2f312ec6fb1487ef42f23fd56a71a6d967bb", size = 4248534, upload-time = "2026-11-16T07:47:11.02Z" }, + { url = "https://files.pythonhosted.org/packages/73/ec/3363dba853a48b1f8881a31f32bdbe562107b5cb9f1817489c8849768223/fensu-0.31.1-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0856d99bcf15e13cf18994482141fd38fc8318c88103489ceb4b7aef4cbed4a5", size = 4321633, upload-time = "2026-11-16T07:47:12.631Z" }, + { url = "https://files.pythonhosted.org/packages/4f/22/959cd2a81bcef8c4f16fb5bc97a97146de2bc26eca76380102b63543bc23/fensu-0.31.1-cp312-abi3-win_amd64.whl", hash = "sha256:bf9ceb9dbe70dcf86500c7af1421c7d7ad545407fb2896f17082cbb58c8e4199", size = 3987258, upload-time = "2026-11-16T07:47:14.85Z" }, ] [[package]] name = "fensu-cli" -version = "0.31.0" +version = "0.31.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/2f/672748107c792850f63af6adaefce85d3145b76de36599e6c0dd6a5939cb/fensu_cli-0.31.0.tar.gz", hash = "sha256:67d3aff62712369de68e9b8d01a9370da8cab2268a35b883eeb1b84728a64561", size = 781412, upload-time = "2026-11-15T12:31:38.26Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/1c/90037dc9ace2ec18c2319c7b674e7f545ae27c029503b513a88e169569cc/fensu_cli-0.31.1.tar.gz", hash = "sha256:151bb605adc375aff6d5622a7c431f9f1ab21c13e14b2d8e53466cab279abeda", size = 781510, upload-time = "2026-11-16T07:47:27.946Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/28/f369973cf3b83d863454737dfd0fc6727f7c78cac86b64a2fb77e0132b16/fensu_cli-0.31.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e914c246f32943f8ceecd359c0d3baf107a156aa8951d09ef6a74b166f4cb09c", size = 8022964, upload-time = "2026-11-15T12:31:25.917Z" }, - { url = "https://files.pythonhosted.org/packages/3e/73/481478bd5192f35627573784a8c4cb3c8b7dfb9ba4e2d693d04949986748/fensu_cli-0.31.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b698b322e27fa7eb83477418329eb9e6b72817ebe36ae7aa7bf50fe1737534ed", size = 7535483, upload-time = "2026-11-15T12:31:27.75Z" }, - { url = "https://files.pythonhosted.org/packages/af/e2/fdcf0db07a5644b0964acc492b316182912f8873f18a19b3a555a79eba03/fensu_cli-0.31.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3fdd19fe970509d8452a8c4e37f0fffdaa85e6e9ec0c0c9054f453ca2dad113d", size = 7894784, upload-time = "2026-11-15T12:31:29.665Z" }, - { url = "https://files.pythonhosted.org/packages/ad/69/87c07af19a513191a326d516541eeff4dea30e022fe1fe0e8fa21af9517b/fensu_cli-0.31.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:de717bc0ad15b147ab7fd31c2add282b446a30eddf72c46910bab6b142ff8f21", size = 8361052, upload-time = "2026-11-15T12:31:31.99Z" }, - { url = "https://files.pythonhosted.org/packages/8a/64/a1551ab57fd25d706fa8f6a4929ba832c7271dad9b9c7a3499ec6398b211/fensu_cli-0.31.0-py3-none-win_amd64.whl", hash = "sha256:66e91cfe28414c87e66c9517945bb93ca02c38273f04f4978604a4cd10cbcba0", size = 7752224, upload-time = "2026-11-15T12:31:35.271Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a8/9722cb3fbe2758057f47dace9aec4a07acbb675325a70f4246d15d6abb4c/fensu_cli-0.31.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:04ae317d5766e2657ca19541d353133f6034856a9a174bcf5dbeef095e5b9cbb", size = 8030279, upload-time = "2026-11-16T07:47:17.105Z" }, + { url = "https://files.pythonhosted.org/packages/65/5d/e9d80075fe599ea1bcada7901fdce1e08bf7bcda02278888577b3058e214/fensu_cli-0.31.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:19917a4ab561eeb286e7f470d27dd32eba9923eb83159b2fd3e59c2b2eccc511", size = 7531943, upload-time = "2026-11-16T07:47:19.115Z" }, + { url = "https://files.pythonhosted.org/packages/c4/2b/295c81cfaa437e76294c068408340469111af61adc48dc03a78776ee2418/fensu_cli-0.31.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8835a543ae77d889a1a289d26fb96e77cd57d0556604b120e65a2cf3854690f7", size = 7891741, upload-time = "2026-11-16T07:47:20.807Z" }, + { url = "https://files.pythonhosted.org/packages/1c/af/4ad64330327111dcd43e63f649c73b3075a75ac149d9d104ec7cd1c3eb3c/fensu_cli-0.31.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b20b5998c42fcec83bbe0427847cc2c99ace47783788ed3a5604ed0a2a54c890", size = 8356970, upload-time = "2026-11-16T07:47:23.129Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/74a1f365e3115d4a75d3f4775ee082b3337024d383c6dca78ff766a7a2b0/fensu_cli-0.31.1-py3-none-win_amd64.whl", hash = "sha256:e6a90efeef30f21dc4a420f40c841232d29f882cb0dd0ede39f4d5c5d9ae788f", size = 7750497, upload-time = "2026-11-16T07:47:25.128Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.31.0" }, + { name = "fensu", specifier = ">=0.31.1" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index befff2abba..78ef21e6ee 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -53,7 +53,8 @@ table more than once. ### delete_insert -Deletes the rows the new delta replaces, then inserts the delta. Requires either `cursor` or `unique_key`. +Deletes the rows the new delta replaces, then inserts the delta. Requires `cursor`, `unique_key` or +`replace_key`. ```sql MODEL ( @@ -74,13 +75,35 @@ Which rows it deletes depends on the model's settings: | Settings | Rows deleted before the insert | |----------|--------------------------------| | `cursor` | Rows whose cursor falls in the replay window | -| `unique_key` | Rows whose key appears in the delta | -| `cursor` and `unique_key` | Rows in the replay window, and rows whose key appears in the delta | +| `unique_key` | Rows whose key appears in the delta. The key must hold one row per value; for groups of rows per value, use `replace_key` | +| `replace_key` | Rows whose `replace_key` values appear in the delta | +| `cursor` and `unique_key` or `replace_key` | Rows in the replay window, and rows whose key appears in the delta | | `incremental_mode microbatch` | Rows in the batch's window; a `unique_key` only matches rows in diff and review | With both a cursor and a key, a row whose cursor value moved out of the window, such as an order updated after its first load, replaces its earlier version instead of adding a second row. +`unique_key` promises one row per key. Use `replace_key` when a key value covers a group of rows +that each load replaces whole, such as the lines of one forecast version: + +```sql +MODEL ( + description "Forecast lines, several per forecast version", + materialized incremental, + incremental_strategy delete_insert, + replace_key [forecast_version], +); + +SELECT forecast_version, product_id, forecast_month, quantity +FROM __ref("stg_forecast_lines") +``` + +A delta holding version `2026-06` deletes every `2026-06` row and inserts the new ones. Other +versions keep their rows. A model sets `unique_key` or `replace_key`, not both, and `replace_key` +is valid only with `delete_insert` and not with `incremental_mode microbatch`. BigQuery rejects +`replace_key` with `A012` when planning; every other adapter replaces groups. Rules that need a +proven key, such as `SQLCLEAN018`, use `unique_key` and never `replace_key`. + ### merge Upserts rows using a unique key. Matched rows are updated; unmatched rows are inserted. diff --git a/website/src/content/docs/docs/concepts/interpolation.mdx b/website/src/content/docs/docs/concepts/interpolation.mdx index ab5bdde0ce..b3ab897980 100644 --- a/website/src/content/docs/docs/concepts/interpolation.mdx +++ b/website/src/content/docs/docs/concepts/interpolation.mdx @@ -25,6 +25,7 @@ The rule is simple: if it's any SQL that will be executed, it uses `@`. If it's | `@name` / `@'name'` | Generic audit SQL | Raw / SQL-literal audit argument | | `${CTX:...}` | TOML/YAML config values | Config compilation | | `${ENV:...}` | TOML/YAML config values | Config compilation | +| `${coalesce(...)}`, `${if(...)}`, `${eq(...)}`, `${ne(...)}` | TOML/YAML config values | Config compilation - see [Config value functions](#config-value-functions) | `@@CTX:` is intentionally SQL-hook-only. Model SQL describes a relation's data and should not reference its own destination identity. SQL hooks are the operational SQL layer where destination context is useful - grants, logging, post-materialization DDL. Python hooks access the same information through `ctx.destination` on the [`HookContext`](/docs/concepts/models/hooks/python#hook-context) object. @@ -92,6 +93,38 @@ WHERE source_system = '@@ENV:SOURCE_SYSTEM' If the environment variable is not set, SQLBuild raises a compile error. +## Config value functions + +A `${...}` config value can choose between values with a function. This target uses +`WAREHOUSE_SCHEMA` when it is set, and builds `DEV_` from `WAREHOUSE_USER` otherwise: + +```toml +[targets.dev] +schema = "${coalesce(ENV:WAREHOUSE_SCHEMA, 'DEV_${ENV:WAREHOUSE_USER}')}" +``` + +| Function | Returns | +|----------|---------| +| `coalesce(a, b, ...)` | The first argument that is set and true; otherwise the last argument | +| `if(condition, a, b)` | `a` when `condition` is true, otherwise `b` | +| `eq(a, b)` | `true` when `a` and `b` read the same as text | +| `ne(a, b)` | `true` when `a` and `b` read differently as text | + +An argument is a project variable (`schema_prefix`), `ENV:NAME`, `CTX:name`, `true`, `false`, +`null`, a quoted string or another function call. A quoted string can hold `${...}` templates, +which render into it as text, as in `'DEV_${ENV:WAREHOUSE_USER}'`. + +Details: + +- An argument is false when it is `null`, empty or blank, `0`, or `false` in any letter case. +- A missing variable makes its argument missing, so `coalesce` moves on to the next argument. When + every argument is missing, SQLBuild reports the missing variable in the last one. +- A backslash in a quoted string keeps the next character as written, so `'\${ENV:USER}'` is the + text `${ENV:USER}`. Inside a TOML or YAML double-quoted value, write the backslash twice: + `"${coalesce('\\${ENV:USER}')}"`. +- A template inside a quoted string can use the same quotes as the string around it. Templates nest + up to eight levels deep; a template nested deeper stays as written text. + ## Context variables Context variables provide access to the current model's destination relation, active target, and run metadata. diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 303297b716..29ae89ce89 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -232,6 +232,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged | `cursor_future_action` | `cap` or `error` when a watermark exceeds `cursor_future_max_distance` | | `cursor_inputs` | Upstream names mapped to cursor columns | | `unique_key` | Row identity. `merge` and `delete_insert` replace rows with a matching key; `append` and microbatch `delete_insert` use it only to match rows in diff and review | +| `replace_key` | `delete_insert` only: replaces every row whose `replace_key` values appear in the delta. Use it instead of `unique_key` when one key value covers several rows | | `incremental_mode` | Set to `microbatch` for batched execution | | `microbatch_strategy` | Required in microbatch mode: `watermark` or `rolling_window` | | `cursor_watermark_mode` | Watermark strategy policy: `all` or `any` | diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 50e70594d4..7e03d7a536 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -230,8 +230,8 @@ max_ranking_order_by = 5 # default Keys come from `unique` column audits and model-level `unique` audits naming a column, when the column is also non-null (a `not_null` audit or `nullable false`), and from the `unique_key` of incremental `merge` models and of `delete_insert` models matched by key without a cursor. - Snapshot, append, microbatch and cursor-ranged `delete_insert` `unique_key`s do not prove one - row per key. Keys are followed through projections, renames, filters, one-to-one joins, + Snapshot, append, microbatch and cursor-ranged `delete_insert` `unique_key`s, and every + `replace_key`, do not prove one row per key. Keys are followed through projections, renames, filters, one-to-one joins, `GROUP BY` grains (named or positional), `SELECT DISTINCT`, `UNION`, and `ROW_NUMBER() ... = 1` deduplication in `QUALIFY` or a later `WHERE`; a `QUALIFY` key holds for later steps, never for the window that defines it. A join that can match a row more than once, diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index 851049c889..75cf8e5121 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -101,7 +101,7 @@ dlt_sources: | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | | `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | | `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | -| `primary_key` / `merge_key` | dlt keys for `merge`. | +| `primary_key` / `merge_key` | dlt keys for `merge`. A `merge_key` replaces every row whose key value arrives, like a SQL model's `delete_insert` with [`replace_key`](/docs/concepts/incremental#delete_insert). | | `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | | `schema` | Per-resource schema override (falls back to the group `schema`). | diff --git a/website/src/content/docs/docs/integrations/ingestr.mdx b/website/src/content/docs/docs/integrations/ingestr.mdx index 7704837775..7b4f1327cc 100644 --- a/website/src/content/docs/docs/integrations/ingestr.mdx +++ b/website/src/content/docs/docs/integrations/ingestr.mdx @@ -92,7 +92,7 @@ The `ingestr` block on a source supports the following fields: | `replace` | Drop and recreate the destination table (default when no strategy is set) | | `append` | Insert all rows without deduplication | | `merge` | Upsert based on `primary_key`, using `incremental_key` for change detection | -| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements | +| `delete+insert` | Delete matching rows by `incremental_key` range, then insert replacements. A SQL model does the same with `delete_insert` and a `cursor`; use [`replace_key`](/docs/concepts/incremental#delete_insert) to replace groups of rows by key value instead | | `truncate+insert` | Truncate destination table, then insert all rows | ## Template variables