diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs index e53f254daf..a482e9ba20 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -28,6 +28,9 @@ pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90; pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203"; +/// A FULL backfill of a non-microbatch model whose SQL reads the cursor intrinsics. +pub(crate) const FULL_BACKFILL_WITHOUT_INTERVAL_CODE: &str = "S204"; + pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; pub(crate) const RETENTION_DECREASE: &str = "decrease"; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs index 3460b890f7..5fab639f0e 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs @@ -1,5 +1,6 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. +use crate::planner_phases::constants::FULL_BACKFILL_WITHOUT_INTERVAL_CODE; use crate::planner_resolution::_helpers::cursor::cursor_model::{ compute_model_cursor_bounds, unbounded, }; @@ -169,6 +170,7 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P let error = error_class(py, ErrorId::PlannerInput)?; let kwargs = PyDict::new(py); let message = if let Some(cause) = cause { + kwargs.set_item("code", FULL_BACKFILL_WITHOUT_INTERVAL_CODE)?; kwargs.set_item("help", "rebuild it with an explicit cursor interval (--start-cursor-ts and --end-cursor-ts, or the -int forms), or make it a microbatch model so a full rebuild can run batch by batch")?; format!( "Model '{name}' was given a FULL backfill ({cause}), but it uses cursor intrinsics and a non-microbatch full rebuild has no cursor interval" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md index 0653c5dd08..beb0a47260 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/incremental.md @@ -263,7 +263,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model planned as a FULL backfill, for example by `--full-refresh`, fails planning with `S204`, naming the cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py index a6f720053c..1e1ff5036b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/own_change_replay/test_own_change_replay.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import subprocess from datetime import date from pathlib import Path from typing import Any @@ -29,6 +31,7 @@ insert_table_between_upstream_and_view, mark_first_day, merge_downstream_files, + plan, plan_models, relation_types, rename_upstream_with_cli, @@ -341,6 +344,7 @@ def test_given_full_refresh_false_caller_when_function_forces_full_rebuild_then_ RefusedPlanTestCase( description="explicit full refresh of a cursor-intrinsic merge names the FULL cause", expected_fragments=( + "error[S204]", "Model 'downstream' was given a FULL backfill (--full-refresh)", "non-microbatch full rebuild has no cursor interval", "--start-cursor-ts", @@ -361,6 +365,11 @@ def test_given_full_refresh_flag_on_cursor_intrinsic_merge_when_planning_then_na args=("--full-refresh", "--select", "downstream"), expected=test_case.expected_fragments, ) + refused: subprocess.CompletedProcess[str] = plan( + project_dir=project_dir, args=("--json", "--full-refresh", "--select", "downstream") + ) + assert refused.returncode != 0, refused.stdout + refused.stderr + assert json.loads(refused.stdout)["error"]["code"] == "S204", refused.stdout + refused.stderr @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 5ab6298183..7e9c286e8b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -106,7 +106,7 @@ def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics( ) -> None: model: CompiledModelView = build_cursor_intrinsic_model(config_values=test_case.model_config) - with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): + with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment) as raised: resolve_model_sql( adapter=DuckDbAdapter(), model=model, @@ -116,3 +116,4 @@ def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics( full_refresh=test_case.full_refresh, cursor_overrides=CursorOverridePair(), ) + assert raised.value.code == "S204" diff --git a/website/src/content/docs/docs/concepts/incremental.mdx b/website/src/content/docs/docs/concepts/incremental.mdx index 78ef21e6ee..51426d8121 100644 --- a/website/src/content/docs/docs/concepts/incremental.mdx +++ b/website/src/content/docs/docs/concepts/incremental.mdx @@ -252,7 +252,7 @@ WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() ``` -`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. +`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model planned as a FULL backfill, for example by `--full-refresh`, fails planning with `S204`, naming the cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it. In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark.