Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90;

pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203";

/// A FULL backfill of a non-microbatch model whose SQL reads the cursor intrinsics.
pub(crate) const FULL_BACKFILL_WITHOUT_INTERVAL_CODE: &str = "S204";

pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301";

pub(crate) const RETENTION_DECREASE: &str = "decrease";
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Model, function and SQL-hook orchestration through adapter-owned rendering.

use crate::planner_phases::constants::FULL_BACKFILL_WITHOUT_INTERVAL_CODE;
use crate::planner_resolution::_helpers::cursor::cursor_model::{
compute_model_cursor_bounds, unbounded,
};
Expand Down Expand Up @@ -169,6 +170,7 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P
let error = error_class(py, ErrorId::PlannerInput)?;
let kwargs = PyDict::new(py);
let message = if let Some(cause) = cause {
kwargs.set_item("code", FULL_BACKFILL_WITHOUT_INTERVAL_CODE)?;
kwargs.set_item("help", "rebuild it with an explicit cursor interval (--start-cursor-ts and --end-cursor-ts, or the -int forms), or make it a microbatch model so a full rebuild can run batch by batch")?;
format!(
"Model '{name}' was given a FULL backfill ({cause}), but it uses cursor intrinsics and a non-microbatch full rebuild has no cursor interval"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ WHERE ordered_at >= __cursor_start()
AND ordered_at < __cursor_end()
```

`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it.
`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model planned as a FULL backfill, for example by `--full-refresh`, fails planning with `S204`, naming the cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it.

In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

from __future__ import annotations

import json
import subprocess
from datetime import date
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -29,6 +31,7 @@
insert_table_between_upstream_and_view,
mark_first_day,
merge_downstream_files,
plan,
plan_models,
relation_types,
rename_upstream_with_cli,
Expand Down Expand Up @@ -341,6 +344,7 @@ def test_given_full_refresh_false_caller_when_function_forces_full_rebuild_then_
RefusedPlanTestCase(
description="explicit full refresh of a cursor-intrinsic merge names the FULL cause",
expected_fragments=(
"error[S204]",
"Model 'downstream' was given a FULL backfill (--full-refresh)",
"non-microbatch full rebuild has no cursor interval",
"--start-cursor-ts",
Expand All @@ -361,6 +365,11 @@ def test_given_full_refresh_flag_on_cursor_intrinsic_merge_when_planning_then_na
args=("--full-refresh", "--select", "downstream"),
expected=test_case.expected_fragments,
)
refused: subprocess.CompletedProcess[str] = plan(
project_dir=project_dir, args=("--json", "--full-refresh", "--select", "downstream")
)
assert refused.returncode != 0, refused.stdout + refused.stderr
assert json.loads(refused.stdout)["error"]["code"] == "S204", refused.stdout + refused.stderr


@pytest.mark.parametrize(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics(
) -> None:
model: CompiledModelView = build_cursor_intrinsic_model(config_values=test_case.model_config)

with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment):
with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment) as raised:
resolve_model_sql(
adapter=DuckDbAdapter(),
model=model,
Expand All @@ -116,3 +116,4 @@ def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics(
full_refresh=test_case.full_refresh,
cursor_overrides=CursorOverridePair(),
)
assert raised.value.code == "S204"
2 changes: 1 addition & 1 deletion website/src/content/docs/docs/concepts/incremental.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ WHERE ordered_at >= __cursor_start()
AND ordered_at < __cursor_end()
```

`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model given a FULL backfill by `--full-refresh` fails with a message naming that cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it.
`__cursor_start()` is the effective inclusive start and `__cursor_end()` is the effective exclusive end after cursor floors, lookback, replay policy, and command-line overrides have been applied. The intrinsics accept no arguments and are only valid in built-in cursor incremental model query SQL. They are rejected in functions, hooks, audits, SQL tests and scenarios, source expressions, non-incremental or cursorless models, custom materializations, and non-microbatch full refreshes. A non-microbatch model planned as a FULL backfill, for example by `--full-refresh`, fails planning with `S204`, naming the cause; pass an explicit interval with `--start-cursor-ts` and `--end-cursor-ts` (or the `-int` forms) to rebuild it.

In microbatch mode, the intrinsics resolve to each batch's concrete bounds. A microbatch full refresh discovers its range from current inputs while ignoring the old destination watermark.

Expand Down
Loading