From 28d12e09fb130cc3b89c677346f3d4c7df470c8d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:57:53 +0000 Subject: [PATCH 1/3] fix: give the refused FULL backfill of a cursor-intrinsic model its own code A FULL backfill of a non-microbatch model whose SQL reads __cursor_start() or __cursor_end() is refused, because a full rebuild has no cursor interval. The refusal used the planner's default code S000, so the plan JSON, the control API and the UI could not tell it apart from any other planner input error. It now fails with S204, listed with the other planner codes, keeping its message and the remedy that names --start-cursor-ts and --end-cursor-ts. --- .../src/bindings/_helpers/planner_phases/constants.rs | 2 ++ .../bindings/_helpers/planner_resolution/orchestration.rs | 2 ++ fensu.toml | 3 ++- .../sqlbuild/references/docs/concepts/incremental.md | 2 +- .../build/own_change_replay/test_own_change_replay.py | 8 ++++++++ website/src/content/docs/docs/concepts/incremental.mdx | 2 +- 6 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index 5c45d15791..5c73a29cbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -8,6 +8,8 @@ pub(crate) const DECREASE_WARNING: &str = "Retention decreases are irreversible once warehouse history expires."; pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90; pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203"; +/// A FULL backfill of a non-microbatch model whose SQL reads the cursor intrinsics. +pub(crate) const FULL_BACKFILL_WITHOUT_INTERVAL_CODE: &str = "S204"; pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; pub(crate) const RETENTION_DECREASE: &str = "decrease"; pub(crate) const RETENTION_INCREASE: &str = "increase"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 9d439d3bb5..9355d9b9a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::model_config::typed_config::{ config_node_of, config_text_is, config_text_object, config_value, model_values, }; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_phases::constants::FULL_BACKFILL_WITHOUT_INTERVAL_CODE; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_model::{ compute_model_cursor_bounds, unbounded, @@ -285,6 +286,7 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P let error = error_class(py, ErrorId::PlannerInput)?; let kwargs = PyDict::new(py); let message = if let Some(cause) = cause { + kwargs.set_item("code", FULL_BACKFILL_WITHOUT_INTERVAL_CODE)?; kwargs.set_item("help", "rebuild it with an explicit cursor interval (--start-cursor-ts and --end-cursor-ts, or the -int forms), or make it a microbatch model so a full rebuild can run batch by batch")?; format!( "Model '{name}' was given a FULL backfill ({cause}), but it uses cursor intrinsics and a non-microbatch full rebuild has no cursor interval" diff --git a/fensu.toml b/fensu.toml index 0868e9b020..67ba42f225 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1681,7 +1681,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] # Raised from 197,891: SQBRPERFORMANCE102 bindings plumbing, +77 with sqlbuild-python-base. -thresholds = { max_crate_code_lines = 197963 } +# Raised from 197,963: the planner code of the FULL backfill refusal, +3. +thresholds = { max_crate_code_lines = 197966 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out. Set after #1514 moved the shared runtime into sqlbuild-python-base, plus 4 lines for the janitor preserve check from #1512." [[targets.rust.threshold_overrides]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index a0ea0cf0da..d4e1bfe6c5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -240,7 +240,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails planning with `S204`, naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py index a6f720053c..ba77d6cc6a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import subprocess from datetime import date from pathlib import Path from typing import Any @@ -29,6 +31,7 @@ insert_table_between_upstream_and_view, mark_first_day, merge_downstream_files, + plan, plan_models, relation_types, rename_upstream_with_cli, @@ -341,6 +344,7 @@ def test_given_full_refresh_false_caller_when_function_forces_full_rebuild_then_ RefusedPlanTestCase( description="explicit full refresh of a cursor-intrinsic merge names the FULL cause", expected_fragments=( + "error[S204]", "Model 'downstream' was given a FULL backfill (--full-refresh)", "non-microbatch full rebuild has no cursor interval", "--start-cursor-ts", @@ -361,6 +365,10 @@ def test_given_full_refresh_flag_on_cursor_intrinsic_merge_when_planning_then_na args=("--full-refresh", "--select", "downstream"), expected=test_case.expected_fragments, ) + refused: subprocess.CompletedProcess[str] = plan( + project_dir=project_dir, args=("--json", "--full-refresh", "--select", "downstream") + ) + assert json.loads(refused.stdout)["error"]["code"] == "S204", refused.stdout + refused.stderr @pytest.mark.parametrize( diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index befff2abba..6d3e72c2c8 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -229,7 +229,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails planning with `S204`, naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. From 8a4e9c3e6abcd27aca328b2123f43336b94d8022 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:03:02 +0000 Subject: [PATCH 2/3] fix: name every FULL backfill cause in the S204 docs and check the code in the unit test --- .../skills/sqlbuild/references/docs/concepts/incremental.md | 2 +- .../main/build/own_change_replay/test_own_change_replay.py | 1 + .../planner/_helpers/resolve/test_cursor_intrinsics.py | 3 ++- website/src/content/docs/docs/concepts/incremental.mdx | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index d4e1bfe6c5..91836b05f7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -240,7 +240,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails planning with `S204`, naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model planned as a FULL backfill, for example by `--full-refresh`, fails planning with `S204`, naming the cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py index ba77d6cc6a..1e1ff5036b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py @@ -368,6 +368,7 @@ def test_given_full_refresh_flag_on_cursor_intrinsic_merge_when_planning_then_na refused: subprocess.CompletedProcess[str] = plan( project_dir=project_dir, args=("--json", "--full-refresh", "--select", "downstream") ) + assert refused.returncode != 0, refused.stdout + refused.stderr assert json.loads(refused.stdout)["error"]["code"] == "S204", refused.stdout + refused.stderr diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 5ab6298183..7e9c286e8b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -106,7 +106,7 @@ def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics( ) -> None: model: CompiledModelView = build_cursor_intrinsic_model(config_values=test_case.model_config) - with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): + with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment) as raised: resolve_model_sql( adapter=DuckDbAdapter(), model=model, @@ -116,3 +116,4 @@ def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics( full_refresh=test_case.full_refresh, cursor_overrides=CursorOverridePair(), ) + assert raised.value.code == "S204" diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index 6d3e72c2c8..91aca75b1c 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -229,7 +229,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails planning with `S204`, naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model planned as a FULL backfill, for example by `--full-refresh`, fails planning with `S204`, naming the cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. From 4d9d4018ce1fc86028873b51d719310dbfd3a6cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:17:32 +0000 Subject: [PATCH 3/3] chore: set the Python bindings cap to its measured size --- fensu.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fensu.toml b/fensu.toml index c4f6c000b9..0f3b112874 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1670,7 +1670,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147789 } +thresholds = { max_crate_code_lines = 147792 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]]