From 4bd5fdaa760efca83f7c8fff5bbeda67f51b11bb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:13:22 +0000 Subject: [PATCH 01/15] wip: carry the active target's settings into Python node, loader and hook contexts Python side only; the native construction is not written yet. --- .../sqlbuild-host/src/entries/_helpers/executor.rs | 5 +++++ crates/sqlbuild-host/src/entries/models.rs | 1 + src/sqlbuild/_native_host.py | 1 + src/sqlbuild/_native_host.pyi | 1 + src/sqlbuild/executor/build/_helpers/source_node.py | 1 + .../executor/build/classes/build_scheduler.py | 1 + src/sqlbuild/executor/build/models.py | 2 ++ src/sqlbuild/executor/load/_helpers/external.py | 1 + .../executor/load/_helpers/loader_invocation.py | 1 + src/sqlbuild/executor/load/models.py | 3 +++ .../executor/python_nodes/_helpers/execution.py | 1 + .../python_nodes/_helpers/ingress_execution.py | 1 + .../executor/python_nodes/_helpers/python_checks.py | 1 + src/sqlbuild/executor/python_nodes/models.py | 3 +++ .../executor/run/_helpers/execution/hook_phases.py | 1 + .../executor/run/_helpers/execution/hooks.py | 1 + src/sqlbuild/executor/run/models.py | 4 ++++ src/sqlbuild/runtime/contracts/models.py | 12 ++++++++++++ 18 files changed, 41 insertions(+) diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 17302822a0..d59f27d80d 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -624,6 +624,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, + HostEntry { + name: "TargetSettings", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, HostEntry { name: "ExecutionResourceKind", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 70f6e3a9ab..1fee19c724 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -329,6 +329,7 @@ pub enum ExecutorHost { PruneResult, ComputeLogStream, ConnectionHooks, + TargetSettings, ExecutionResourceKind, ExecutionHistoryStorageError, IntegrityConflictError, diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 46a0ef8207..11b8f0ec9e 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -554,6 +554,7 @@ "PruneResult": ("sqlbuild.runtime.compute_logs.models", "PruneResult"), "ComputeLogStream": ("sqlbuild.runtime.compute_logs.types", "ComputeLogStream"), "ConnectionHooks": ("sqlbuild.runtime.contracts.models", "ConnectionHooks"), + "TargetSettings": ("sqlbuild.runtime.contracts.models", "TargetSettings"), "ExecutionResourceKind": ("sqlbuild.runtime.contracts.types", "ExecutionResourceKind"), "ExecutionHistoryStorageError": ( "sqlbuild.runtime.execution_history.exceptions", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index c3d24a82bd..34fc9f956d 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -518,6 +518,7 @@ from sqlbuild.runtime.compute_logs.models import FinalCaptureMetadata as FinalCa from sqlbuild.runtime.compute_logs.models import PruneResult as PruneResult from sqlbuild.runtime.compute_logs.types import ComputeLogStream as ComputeLogStream from sqlbuild.runtime.contracts.models import ConnectionHooks as ConnectionHooks +from sqlbuild.runtime.contracts.models import TargetSettings as TargetSettings from sqlbuild.runtime.contracts.types import ExecutionResourceKind as ExecutionResourceKind from sqlbuild.runtime.event_exporting.exceptions import ( EventExporterInputError as EventExporterInputError, diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 84ae9ffec6..146df17c03 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -72,6 +72,7 @@ def execute_build_source_node( runtime=LoadRuntimeParams( run_id=runtime.run_id, target=runtime.target, + target_config=runtime.target_config, vars=runtime.effective_vars or {}, is_reload=runtime.loader_is_reload, runtime_dir=runtime.runtime_dir, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 5f55987d43..c5bbebbbf4 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1210,6 +1210,7 @@ def _execute_model_node( hook_functions=self._plan.hook_functions, enforce_explicit_references=self._plan.enforce_explicit_references, effective_target_name=self._target, + target_config=self._runtime.target_config, effective_vars=self._effective_vars, providers=self._providers, python_identity_recorder=self._python_identity_recorder, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index ece162eb01..cd181d2bdc 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -34,6 +34,7 @@ from sqlbuild.microbatches.types import MicrobatchEventStore from sqlbuild.provider.main.runtime import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.runtime.contracts.models import TargetSettings from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, NodeStartCallback from sqlbuild.runtime.execution_limits.models import RunItemTimeouts @@ -78,6 +79,7 @@ class BuildRuntimeParams: start_cursor_int: int | None = None end_cursor_int: int | None = None target: str = "" + target_config: TargetSettings | None = None effective_vars: dict[str, object] | None = None use_color: bool = False providers: ProviderContainer | None = None diff --git a/src/sqlbuild/executor/load/_helpers/external.py b/src/sqlbuild/executor/load/_helpers/external.py index b0f4773988..c85d333541 100644 --- a/src/sqlbuild/executor/load/_helpers/external.py +++ b/src/sqlbuild/executor/load/_helpers/external.py @@ -56,6 +56,7 @@ def execute_external_source_load( run_id=runtime.run_id, runtime_dir=runtime.runtime_dir, target=runtime.target, + target_config=runtime.target_config, vars=runtime.vars, is_reload=runtime.is_reload, use_color=runtime.use_color, diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index d287c8681c..3ee4a427b4 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -94,6 +94,7 @@ def build_loader_context( run_id=runtime.run_id, runtime_dir=runtime.runtime_dir, target=runtime.target, + target_config=runtime.target_config, vars=runtime.vars, is_reload=runtime.is_reload, use_color=runtime.use_color, diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 857aff8e6f..ec4ef6be0b 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -27,6 +27,7 @@ from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind +from sqlbuild.runtime.contracts.models import TargetSettings from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, ExecutionResourceKind @@ -149,6 +150,7 @@ class LoaderContext(DependencyResultReader): runtime_dir: Path = Path("target") on_progress: Callable[[str], None] | None = None relation_guard: RuntimeRelationGuard | None = field(default=None, repr=False) + target_config: TargetSettings | None = None def execute_sql(self, sql: str) -> Any: if self.relation_guard is not None: @@ -281,6 +283,7 @@ class LoadRuntimeParams: target: str | None vars: dict[str, object] is_reload: bool + target_config: TargetSettings | None = None runtime_dir: Path = Path("target") start_cursor_ts: datetime | None = None end_cursor_ts: datetime | None = None diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index dafc8e882a..194ea2ca61 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -422,6 +422,7 @@ def _build_context( statement_recorder=statement_recorder, run_state=run_state, result_store=result_store, + target_config=runtime.target_config, default_database=runtime.default_database, default_schema=runtime.default_schema, relation_targets=runtime.resolved_relation_targets, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 6a24e47e14..d364544c48 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -283,6 +283,7 @@ def _execute_ingress_loader( runtime=LoadRuntimeParams( run_id=runtime.run_id, target=runtime.target, + target_config=runtime.target_config, vars=runtime.vars, is_reload=runtime.is_reload, start_cursor_ts=runtime.start_cursor_ts, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 220af44090..508a4b4d03 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -165,6 +165,7 @@ def _execute_one_python_check( statement_recorder=StatementRecorder(), run_state=run_state, result_store=result_store, + target_config=runtime.target_config, default_database=runtime.default_database, default_schema=runtime.default_schema, relation_targets=runtime.resolved_relation_targets, diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 4d7ffae694..1c794faf75 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -30,6 +30,7 @@ from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity +from sqlbuild.runtime.contracts.models import TargetSettings from sqlbuild.runtime.contracts.types import NodeStartCallback @@ -134,6 +135,7 @@ class PythonNodeRuntime: target: str | None vars: dict[str, object] is_reload: bool + target_config: TargetSettings | None = None default_database: str | None = None default_schema: str | None = None relation_targets: dict[SqlResourceRef, str] | None = None @@ -178,6 +180,7 @@ class BasePythonNodeContext(DependencyResultReader): statement_recorder: StatementRecorder run_state: PythonNodeRunState | None = None result_store: Any | None = None + target_config: TargetSettings | None = None default_database: str | None = None default_schema: str | None = None relation_targets: dict[SqlResourceRef, str] = field(default_factory=dict) diff --git a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py index 6841a6b8f4..3cf0b6b512 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py @@ -132,6 +132,7 @@ def build_model_hook_run( destination=context.entry.destination, run_id=context.run_id, target=context.effective_target_name, + target_config=context.target_config, effective_vars=context.effective_vars if effective_vars is None else effective_vars, statement_recorder=statement_recorder, providers=context.providers, diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index a83dee9f83..5974507a52 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -466,6 +466,7 @@ def build_hook_context( # noqa: PLR0913 hook_index=invocation.index, run_id=hook_run.run_id, target=hook_run.target, + target_config=hook_run.target_config, vars=hook_run.effective_vars if hook_run.effective_vars is not None else {}, destination=relation, adapter_name=adapter.adapter_name, diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 551539f588..9177b7d6b5 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -56,6 +56,7 @@ from sqlbuild.microbatches.types import MicrobatchEventStore from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.runtime.contracts.models import TargetSettings @dataclass(frozen=True, kw_only=True, init=False) @@ -104,6 +105,7 @@ class HookContext: providers: ProviderContainer = field(default_factory=_empty_provider_container, repr=False) relations: Mapping[SqlResourceRef, str] = field(default_factory=dict, repr=False) relation_guard: RuntimeRelationGuard | None = field(default=None, repr=False) + target_config: TargetSettings | None = None def execute_sql(self, sql: str) -> None: if self.relation_guard is not None: @@ -314,6 +316,7 @@ class ModelMaterializationContext: schema_prepared: bool = False hook_functions: tuple[DiscoveredHookFunction, ...] = () effective_target_name: str | None = None + target_config: TargetSettings | None = None effective_vars: Mapping[str, object] | None = None providers: ProviderContainer | None = None python_identity_recorder: PythonIdentityRecorder | None = None @@ -532,6 +535,7 @@ class HookRunContext: destination: CompiledRelationLocation | None = None run_id: str = "" target: str | None = None + target_config: TargetSettings | None = None effective_vars: Mapping[str, object] | None = None statement_recorder: StatementRecorder | None = None providers: ProviderContainer | None = None diff --git a/src/sqlbuild/runtime/contracts/models.py b/src/sqlbuild/runtime/contracts/models.py index ed2c0f8d6a..e0005f48cd 100644 --- a/src/sqlbuild/runtime/contracts/models.py +++ b/src/sqlbuild/runtime/contracts/models.py @@ -8,6 +8,18 @@ from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback +@dataclass(frozen=True) +class TargetSettings: + """The active target's database and schemas, resolved as SQLBuild uses them.""" + + name: str | None + database: str | None + schema: str | None + loader_schema: str | None + database_preserved: bool = False + schema_preserved: bool = False + + @dataclass(frozen=True) class ConnectionHooks: """Progress and connection lifecycle callbacks for long-running operations.""" From 3ba687f890c3954fe297ee36e2da878188f778bd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:32:05 +0000 Subject: [PATCH 02/15] fix: stop a concurrent build when an error escapes a node At concurrency 1, an error that escapes a node's execution, such as a worker report the control API rejects or a run cancellation raised from the node-start report, stops the build. Above concurrency 1 the worker wrapper turned the same error into a failed model result, so the build went on starting later nodes, including after the server had rejected the run's state. Concurrent workers now pass the escaped error back to the scheduler, which stops dispatching, lets the nodes already running finish, and raises the error as the serial path does. --- .../executor/build/_helpers/scheduler.py | 11 +--- .../executor/build/classes/build_scheduler.py | 12 +++- src/sqlbuild/executor/build/models.py | 6 +- .../publication/test_worker.py | 58 +++++++++++++++++++ 4 files changed, 74 insertions(+), 13 deletions(-) diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 18f57fbb91..4522c7d643 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -24,7 +24,6 @@ from sqlbuild.executor.build.constants import ( BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, BUILD_SOURCE_FRESHNESS_BLOCKED_CODE, - BUILD_WORKER_FAILED_CODE, INCREMENTAL_ACTIONS, ) from sqlbuild.executor.build.models import ( @@ -68,15 +67,7 @@ def _build_worker_success_completion( def _build_worker_failure_completion( *, key: CompiledObjectKey, error: BaseException ) -> NodeCompletion: - return NodeCompletion( - key=key, - result=ModelExecutionResult( - model_name=key.name, - status=ExecutionStatus.FAILED, - error_code=BUILD_WORKER_FAILED_CODE, - error_message=str(error), - ), - ) + return NodeCompletion(key=key, error=error) def _dispatch_model( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 5f55987d43..97071a08a2 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -453,6 +453,7 @@ def _run_serial(self) -> None: self._connection_pool.put(connection) def _run_concurrent(self) -> None: + escaped: BaseException | None = None with ThreadPoolExecutor(max_workers=self._max_concurrency) as pool: self._report_scheduler_state() while self._ready or self._in_flight: @@ -518,8 +519,17 @@ def _run_concurrent(self) -> None: completion: NodeCompletion = self._completion_queue.get() self._in_flight.discard(completion.key) - self._handle_completion(key=completion.key, result=completion.result) + # An error that escaped a node stops the run as it does at concurrency 1, once + # the nodes already running have finished. + if completion.error is not None: + escaped = escaped or completion.error + self._stop = True + continue + if completion.result is not None: + self._handle_completion(key=completion.key, result=completion.result) self._report_scheduler_state() + if escaped is not None: + raise escaped def _report_scheduler_state(self) -> None: callback: Callable[[SchedulerState], None] | None = self._callbacks.on_scheduler_state diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index ece162eb01..64d27f05aa 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -196,7 +196,7 @@ class BuildIndexes: @dataclass(frozen=True) class NodeCompletion: - """Result of a single node execution passed back from a worker to the scheduler.""" + """A node's result, or the error that escaped it, passed back from a worker.""" key: CompiledObjectKey result: ( @@ -205,7 +205,9 @@ class NodeCompletion: | FunctionExecutionResult | SqlTestExecutionResult | LoadExecutionResult - ) + | None + ) = None + error: BaseException | None = None @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 4b469b733a..4435edffa1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -1454,6 +1454,64 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "a rejected start report stops the run at concurrency 2", + "SELECT count(*) AS amount FROM range(200000000) t(value) WHERE value % 7 = 0", + "SELECT table_name FROM duckdb_tables() " + "WHERE table_name IN ('orders', 'refunds', 'returns') ORDER BY table_name", + "partial", + [("orders",)], + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\nconcurrency = 2\n' + ), + "models/refunds.sql": ( + "MODEL (description 'Refunds.', materialized table);\nSELECT 2 AS amount\n" + ), + "models/returns.sql": ( + "MODEL (description 'Returns.', materialized table);\n" + 'SELECT amount FROM __ref("orders")\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders, refunds and returns."\n' + 'select = "orders refunds returns"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + ), + }, + store_setup=( + "CREATE TRIGGER test_reject_refunds_start BEFORE UPDATE OF started_at " + "ON sqb_run_nodes WHEN NEW.node_key = 'model:refunds' " + "AND NEW.started_at IS NOT NULL BEGIN " + "SELECT RAISE(ABORT, 'synthetic rejection of the refunds start'); END;\n" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_rejected_node_report_when_concurrent_then_the_run_stops_before_later_nodes( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + nodes: list[tuple[str, str, bool]] = control.execute( + "SELECT node_key, status, started_at IS NOT NULL FROM sqb_run_nodes " + "WHERE run_id=? AND kind='node' ORDER BY node_key", + (run["run_id"],), + ).fetchall() + assert ("model:returns", "skipped", False) in nodes, (nodes, worker.log.read_text()) + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ From 4551bc3d1c3369e4692a9fbecc1bae4fc056a746 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:41:22 +0000 Subject: [PATCH 03/15] fix: keep the first escaped error explicitly and widen the test's race margin --- src/sqlbuild/executor/build/classes/build_scheduler.py | 3 ++- .../compiler/compiled_project/publication/test_worker.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 97071a08a2..93743a7148 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -522,7 +522,8 @@ def _run_concurrent(self) -> None: # An error that escaped a node stops the run as it does at concurrency 1, once # the nodes already running have finished. if completion.error is not None: - escaped = escaped or completion.error + if escaped is None: + escaped = completion.error self._stop = True continue if completion.result is not None: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 4435edffa1..344395a439 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -1459,7 +1459,7 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s [ WorkerExecutionTestCase( "a rejected start report stops the run at concurrency 2", - "SELECT count(*) AS amount FROM range(200000000) t(value) WHERE value % 7 = 0", + "SELECT count(*) AS amount FROM range(500000000) t(value) WHERE value % 7 = 0", "SELECT table_name FROM duckdb_tables() " "WHERE table_name IN ('orders', 'refunds', 'returns') ORDER BY table_name", "partial", From 6877feeaae9c279a6046b37c128633d1c0bd769e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:42:22 +0000 Subject: [PATCH 04/15] refactor: build the escaped-error completion with NodeCompletion directly The failure builder only forwarded its arguments, so passing the dataclass keeps the runtime package within its code-line cap. --- src/sqlbuild/executor/build/_helpers/scheduler.py | 6 ------ src/sqlbuild/executor/build/classes/build_scheduler.py | 7 ++----- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 4522c7d643..727e48f49e 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -64,12 +64,6 @@ def _build_worker_success_completion( return NodeCompletion(key=key, result=result) -def _build_worker_failure_completion( - *, key: CompiledObjectKey, error: BaseException -) -> NodeCompletion: - return NodeCompletion(key=key, error=error) - - def _dispatch_model( *, context: ModelMaterializationContext, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 93743a7148..a2aa5be483 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -64,7 +64,6 @@ reconcile_model_retention, ) from sqlbuild.executor.build._helpers.scheduler import ( - _build_worker_failure_completion, _build_worker_success_completion, _dispatch_model, _model_execution_resource_kind, @@ -519,8 +518,6 @@ def _run_concurrent(self) -> None: completion: NodeCompletion = self._completion_queue.get() self._in_flight.discard(completion.key) - # An error that escaped a node stops the run as it does at concurrency 1, once - # the nodes already running have finished. if completion.error is not None: if escaped is None: escaped = completion.error @@ -559,7 +556,7 @@ def _worker(self, key: CompiledObjectKey) -> None: completion_queue=self._completion_queue, execute=lambda connection: self._execute_node(key=key, connection=connection), build_success=_build_worker_success_completion, - build_failure=_build_worker_failure_completion, + build_failure=NodeCompletion, ) def _is_concurrent_microbatch(self, key: CompiledObjectKey) -> bool: @@ -586,7 +583,7 @@ def _concurrent_microbatch_worker( key=key, pool=pool, connection=connection ), build_success=_build_worker_success_completion, - build_failure=_build_worker_failure_completion, + build_failure=NodeCompletion, ) def _execute_concurrent_microbatch_node( From 483a5e7b9e7ed6c8f34cda77f1a86602b7011845 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:02:15 +0000 Subject: [PATCH 05/15] feat: build the active target's settings natively for node, hook and loader contexts Build, check and load resolve TargetSettings once from the target they run against: database and schema as models resolve them, None with a preserved flag for preserve, and the schema a managed source without its own loads into. --- .../_helpers/build_command/execution.rs | 17 ++ .../_helpers/build_command/lifecycle.rs | 8 + .../bindings/_helpers/check/check_command.rs | 10 + .../src/bindings/_helpers/check/check_core.rs | 10 +- .../_helpers/compile_driver/targets.rs | 171 ++++++++++++++++-- .../bindings/_helpers/load/load_command.rs | 9 + src/sqlbuild/_native.pyi | 3 +- 7 files changed, 210 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 59374d9439..d827946302 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::check::check_core::{ use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; +use crate::bindings::_helpers::compile_driver::targets::project_target_settings; use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; use crate::bindings::_helpers::consumption::recorder::{ ConsumptionRecorder, Prepared, held_back_text, prepare as prepare_consumption, run_reason_text, @@ -357,6 +358,14 @@ pub(crate) fn execute_build_plan<'py>( preparation.concurrency.into_pyobject(py)?.into_any(), ), ("loader_is_reload", request("reload_sources")?), + ( + "target_config", + project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&connection_config, &adapter), + )?, + ), ("start_cursor_ts", start_ts.clone()), ("end_cursor_ts", end_ts.clone()), ("start_cursor_int", start_int.clone()), @@ -634,6 +643,14 @@ fn check_runtime<'py>( "is_reload", native_field(&context.request, "reload_sources")?, ), + ( + "target_config", + project_target_settings( + &context.get("discovered_inputs")?, + &project, + (&context.get("connection_config")?, &adapter), + )?, + ), ( "default_database", adapter.call_method0("default_database")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 8ab3d82fbd..58fde65b3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -14,6 +14,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; +use crate::bindings::_helpers::compile_driver::targets::project_target_settings; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::planner_scope::loaders::build_intermediate_source_map; @@ -259,6 +260,7 @@ struct RuntimeParts<'a, 'py> { project: &'a Bound<'py, PyAny>, reload: bool, defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), + target_config: Bound<'py, PyAny>, relations: (Bound<'py, PyAny>, Bound<'py, PyAny>), inputs: &'a LifecycleInputs<'py>, } @@ -282,6 +284,7 @@ fn node_runtime<'py>( ), ("vars", native_field(parts.project, "effective_vars")?), ("is_reload", boolean(py, parts.reload)), + ("target_config", parts.target_config.clone()), ("default_database", parts.defaults.0.clone()), ("default_schema", parts.defaults.1.clone()), ("start_cursor_ts", start_ts.clone()), @@ -441,6 +444,11 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( adapter.call_method0("default_schema") })?, ), + target_config: project_target_settings( + &discovered, + &project, + (&connection_config, &adapter), + )?, relations: (relation_targets, project_relations), inputs, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 0982412eeb..9731d0d303 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pi use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; +use crate::bindings::_helpers::compile_driver::targets::project_target_settings; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; @@ -70,6 +71,7 @@ struct Prepared<'py> { relation_targets: Bound<'py, PyAny>, relation_refs: Bound<'py, PyAny>, defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), + target_config: Bound<'py, PyAny>, project_relations: Bound<'py, PyAny>, } @@ -354,7 +356,13 @@ fn prepare<'py>( )?), )?, )?; + let target_config: Bound<'py, PyAny> = project_target_settings( + &discovered, + &project, + (&native_field(invocation, "connection_config")?, &adapter), + )?; Ok(Prepared { + target_config, check_functions: PyTuple::new(py, check_functions)?.into_any(), lifecycle_plan, relation_targets, @@ -405,6 +413,7 @@ fn node_runtime<'py>( ("target", native_field(&project, "effective_target_name")?), ("vars", native_field(&project, "effective_vars")?), ("is_reload", PyBool::new(py, false).to_owned().into_any()), + ("target_config", prepared.target_config.clone()), ("default_database", prepared.defaults.0.clone()), ("default_schema", prepared.defaults.1.clone()), ("relation_targets", prepared.relation_targets.clone()), @@ -493,6 +502,7 @@ fn run_checks<'py>( validation_refs: prepared.relation_refs.clone(), project_relations: prepared.project_relations.clone(), providers: providers.clone(), + target_config: prepared.target_config.clone(), })?; record_run_state( invocation, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 15d6903129..6cbfd3a328 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -38,7 +38,12 @@ const READ_SIDE_OPTIONS: (&[&str], &[&str]) = ( "lifecycle_plan", "relation_targets", ], - &["validation_refs", "project_relations", "providers"], + &[ + "validation_refs", + "project_relations", + "providers", + "target_config", + ], ); fn user_error(py: Python<'_>, message: String) -> PyErr { @@ -263,6 +268,7 @@ pub(crate) struct ReadSideInputs<'py> { pub(crate) validation_refs: Bound<'py, PyAny>, pub(crate) project_relations: Bound<'py, PyAny>, pub(crate) providers: Bound<'py, PyAny>, + pub(crate) target_config: Bound<'py, PyAny>, } fn reference_kind(reference: &Bound<'_, PyAny>) -> PyResult { @@ -454,6 +460,7 @@ fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult ("target", native_field(&project, "effective_target_name")?), ("vars", native_field(&project, "effective_vars")?), ("is_reload", false.into_pyobject(py)?.to_owned().into_any()), + ("target_config", inputs.target_config.clone()), ( "default_database", inputs.adapter.call_method0("default_database")?, @@ -541,6 +548,7 @@ fn run_check_read_side_dependencies<'py>( validation_refs: arguments.value("validation_refs")?, project_relations: arguments.value("project_relations")?, providers: arguments.value("providers")?, + target_config: arguments.value("target_config")?, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index e665d8bde7..37c2c0e772 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -5,7 +5,7 @@ use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyString, PyStringMethods}; +use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::target_checks::constants::{ @@ -41,14 +41,21 @@ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databrick const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; -use crate::bindings::_helpers::compile_attachment::target::resolved_target; +use crate::bindings::_helpers::compile_attachment::target::{ + build_effective_target_namespace, resolved_target, +}; +use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::{ErrorId, RecordId}; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; @@ -342,7 +349,10 @@ fn target_namespaces( let mut namespaces: Vec = Vec::new(); for source_entry in managed_sources { let namespace: Namespace = managed_source_namespace( - source_entry, + ( + native_field(source_entry, "database")?, + native_field(source_entry, "schema")?, + ), (&target_config, &connection, &effective_vars), adapter, )?; @@ -366,28 +376,28 @@ fn first_truthy<'py>( fallback() } -/// `_managed_source_namespace`: the database and schema a managed source loads into. -fn managed_source_namespace( - source_entry: &Bound<'_, PyAny>, +/// `_managed_source_namespace`: the database and schema a managed source with these own values loads into. +fn managed_source_namespace<'py>( + (source_database, source_schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), (target_config, connection, effective_vars): ( - &Bound<'_, PyAny>, - &Bound<'_, PyAny>, - &Bound<'_, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, ), - adapter: &Bound<'_, PyAny>, + adapter: &Bound<'py, PyAny>, ) -> PyResult { - let py: Python<'_> = source_entry.py(); - let database: Bound<'_, PyAny> = first_truthy( + let py: Python<'py> = adapter.py(); + let database: Bound<'py, PyAny> = first_truthy( &[ - native_field(source_entry, "database")?, + source_database, native_field(target_config, "database")?, mapping_get(connection, "database")?, ], || adapter.call_method0("default_database"), )?; - let schema: Bound<'_, PyAny> = first_truthy( + let schema: Bound<'py, PyAny> = first_truthy( &[ - native_field(source_entry, "schema")?, + source_schema, native_field(target_config, "loader_schema")?, native_field(target_config, "schema")?, mapping_get(connection, "schema")?, @@ -408,6 +418,135 @@ fn managed_source_namespace( Ok((part("database")?, part("schema")?)) } +/// The active target as `TargetSettings`: `database` and `schema` as models resolve them, `None` with a flag when `preserve`, and the schema a managed source without its own loads into. +pub(crate) fn target_settings<'py>( + (discovered, target_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + (connection, effective_vars, adapter): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let target_config: Bound<'py, PyAny> = if target_name.is_none() { + new_record(py, RecordId::TargetConfig, (), None)? + } else { + resolve_target_config( + py, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + target_name, + )? + }; + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let (_, loader_schema): Namespace = managed_source_namespace( + (none.clone(), none.clone()), + (&target_config, connection, effective_vars), + adapter, + )?; + let preserved = |value: &Bound<'py, PyAny>| -> PyResult { + Ok(text_or_none(value)?.as_deref() == Some(PRESERVE_TARGET_VALUE)) + }; + let (database_preserved, schema_preserved) = (preserved(&database)?, preserved(&schema)?); + call_with( + &host_entry(py, ExecutorHost::TargetSettings)?, + &[ + ("name", target_name.clone()), + ( + "database", + if database_preserved { + none.clone() + } else { + database + }, + ), + ( + "schema", + if schema_preserved { + none.clone() + } else { + schema + }, + ), + ( + "loader_schema", + loader_schema + .filter(|value| value != PRESERVE_TARGET_VALUE) + .into_bound_py_any(py)?, + ), + ( + "database_preserved", + PyBool::new(py, database_preserved).to_owned().into_any(), + ), + ( + "schema_preserved", + PyBool::new(py, schema_preserved).to_owned().into_any(), + ), + ], + ) +} + +/// `target_settings` of a compiled project's target. +pub(crate) fn project_target_settings<'py>( + discovered: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + (connection, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + target_settings( + (discovered, &native_field(project, "effective_target_name")?), + ( + native_field(project, "effective_target_database")?, + native_field(project, "effective_target_schema")?, + ), + ( + connection, + &native_field(project, "effective_vars")?, + adapter, + ), + ) +} + +/// `target_settings` of `selected_target` for a command that does not compile, such as `load`. +pub(crate) fn selected_target_settings<'py>( + (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter_name: &str, + (connection, effective_vars, adapter): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let connection_database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name) { + py.None().into_bound(py) + } else { + mapping_get(connection, "database")? + }; + let namespace: Bound<'py, PyTuple> = build_effective_target_namespace( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", selected_target.clone()), + ("connection_database", connection_database), + ("connection_schema", mapping_get(connection, "schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ("default_schema", adapter.call_method0("default_schema")?), + ], + )?), + )?; + target_settings( + (discovered, &namespace.get_item(0)?), + (namespace.get_item(2)?, namespace.get_item(3)?), + (connection, effective_vars, adapter), + ) +} + /// A managed source's `database` and `schema` with project vars expanded and no context templates. pub(crate) fn expanded_loader_namespace<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 4169796e01..b1c88db5d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -22,6 +22,7 @@ use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionPro use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::targets::selected_target_settings; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -79,6 +80,7 @@ struct Preparation<'py> { cursor_overrides: Bound<'py, PyAny>, provider_session: Bound<'py, PyAny>, project_relations: Bound<'py, PyAny>, + target_config: Bound<'py, PyAny>, } fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { @@ -361,7 +363,13 @@ fn prepare<'py>( native_field(discovered, "providers")?, )], )?; + let target_config: Bound<'py, PyAny> = selected_target_settings( + (discovered, &native_field(request, "selected_target")?), + &adapter_name.extract::()?, + (&connection_config, &runtime.1, &adapter), + )?; Ok(Preparation { + target_config, adapter_name: adapter_name.extract()?, project_relations: project_relations(invocation, &adapter)?, adapter, @@ -395,6 +403,7 @@ fn runtime_params<'py>( ("target", target_name.clone()), ("vars", effective_vars.clone()), ("is_reload", native_field(request, "reload")?), + ("target_config", preparation.target_config.clone()), ( "start_cursor_ts", cursor_timestamp(&native_field(overrides, "start_ts")?)?, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9f0fe7bc80..da45f67a47 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -131,7 +131,7 @@ from sqlbuild.runtime.compute_logs.models import ( PruneResult, ) from sqlbuild.runtime.compute_logs.types import ComputeLogStream -from sqlbuild.runtime.contracts.models import ConnectionHooks +from sqlbuild.runtime.contracts.models import ConnectionHooks, TargetSettings from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.execution_history.models import ( EventFilter, @@ -2281,6 +2281,7 @@ def run_check_read_side_dependencies( validation_refs: frozenset[SqlResourceRef] | None = None, project_relations: dict[SqlResourceRef, str] | None = None, providers: ProviderContainer | None = None, + target_config: TargetSettings | None = None, ) -> tuple[PythonNodeExecutionResult, ...]: ... def finalize_build_cost(finalization: BuildCostFinalization) -> CostRunRecord | None: ... def format_cost_breakdown( From 4c58c2f2ca2fdc05486911a0d9c58e079d0802ee Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:08:41 +0000 Subject: [PATCH 06/15] test: Python tasks, loaders and checks see the locally overridden target settings --- .../cli/commands/main/build/_test_types.py | 10 ++ .../cli/commands/main/build/test_build.py | 122 ++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 56620025a9..a8eb67bede 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1632,3 +1632,13 @@ class PythonUpstreamResultE2ETestCase: select: str upstream_select: str expected_error: str + + +@dataclass(frozen=True) +class PythonTargetConfigBuildE2ETestCase: + """Test case for the active target's settings seen by Python nodes and loaders.""" + + description: str + project_target_lines: str + local_target_lines: str + expected_settings: dict[str, object] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index d788ead43c..7bd24c0417 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -20,6 +20,7 @@ PythonLoaderPersistedResultBuildE2ETestCase, PythonLoaderStatusResultBuildE2ETestCase, PythonPersistedResultBuildE2ETestCase, + PythonTargetConfigBuildE2ETestCase, PythonTargetIsolationBuildE2ETestCase, SelectionAwareStalenessBuildE2ETestCase, ) @@ -1808,6 +1809,127 @@ def test_given_loader_status_result_when_building_then_persists_loader_status_ro ) +_TARGET_CONFIG_RECORDER: str = ( + "import dataclasses, json\n" + "from pathlib import Path\n\n" + "def record(ctx, node):\n" + " line = {'node': node, **dataclasses.asdict(ctx.target_config)}\n" + " with Path(__file__).parents[1].joinpath('target_config.jsonl').open('a') as out:\n" + " out.write(json.dumps(line) + '\\n')\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTargetConfigBuildE2ETestCase( + description="local override of schema and loader_schema", + project_target_lines='schema = "dev"\nloader_schema = "raw_dev"\n', + local_target_lines='schema = "dev_local"\nloader_schema = "raw_local"\n', + expected_settings={ + "name": "dev", + "database": None, + "schema": "dev_local", + "loader_schema": "raw_local", + "database_preserved": False, + "schema_preserved": False, + }, + ), + PythonTargetConfigBuildE2ETestCase( + description="loader_schema falls back to the target schema", + project_target_lines='schema = "dev"\n', + local_target_lines='schema = "dev_local"\n', + expected_settings={ + "name": "dev", + "database": None, + "schema": "dev_local", + "loader_schema": "dev_local", + "database_preserved": False, + "schema_preserved": False, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_local_target_override_when_running_python_then_every_context_sees_resolved_target( + test_case: PythonTargetConfigBuildE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_target_config_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "python_target_config_project"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "python_target_config_project.duckdb"\n\n' + "[targets.dev]\n" + test_case.project_target_lines + ), + "python/recorder.py": _TARGET_CONFIG_RECORDER, + "python/tasks/configured.py": ( + "from python.recorder import record\n" + "from sqlbuild.tasks import task\n\n" + "@task\n" + "def configured_task(ctx):\n" + " '''Test task configured_task.'''\n" + " record(ctx, 'task')\n" + " return ctx.result()\n" + ), + "python/loaders/raw.py": ( + "from python.recorder import record\n" + "from sqlbuild.loaders import loader\n\n" + "@loader\n" + "def raw_orders(ctx):\n" + " '''Test loader raw_orders.'''\n" + " record(ctx, 'loader')\n" + " return [{'order_id': 1}]\n" + ), + "python/checks/configured.py": ( + "from python.recorder import record\n" + "from python.tasks.configured import configured_task\n" + "from sqlbuild.checks import check\n\n" + "@check(depends_on=configured_task)\n" + "def configured_check(ctx):\n" + " '''Test check configured_check.'''\n" + " record(ctx, 'check')\n" + " return ctx.pass_(message='recorded')\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " managed: true\n" + " write_strategy: table\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + ), + }, + ) + (project_dir / "sqlbuild_local.toml").write_text( + "[targets.dev]\n" + test_case.local_target_lines, encoding="utf-8" + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("--no-color", *command), project_dir=project_dir) + for command in ( + ("build", "--select", "configured_task", "raw_orders"), + ("load", "--select", "raw_orders"), + ("check", "--select", "configured_check"), + ) + ] + + for result in results: + assert result.returncode == 0, result.stdout + result.stderr + recorded: list[dict[str, object]] = [ + json.loads(line) + for line in (project_dir / "target_config.jsonl").read_text(encoding="utf-8").splitlines() + ] + assert {str(line.pop("node")) for line in recorded} == {"task", "loader", "check"} + assert all(line == test_case.expected_settings for line in recorded), recorded + + @pytest.mark.parametrize( "test_case", [ From 2552ea11d888bf08d903fc9b37bc8d7a09a8bf4b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:10:15 +0000 Subject: [PATCH 07/15] docs: document ctx.target_config on node, loader and hook contexts --- .../docs/concepts/models/hooks/python.md | 1 + .../docs/concepts/python-nodes/loaders.md | 1 + .../docs/concepts/python-nodes/overview.md | 25 +++++++++++++++++++ .../docs/concepts/models/hooks/python.mdx | 1 + .../docs/concepts/python-nodes/loaders.mdx | 1 + .../docs/concepts/python-nodes/overview.mdx | 25 +++++++++++++++++++ 6 files changed, 54 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md index 62f69e0b13..d2ad1a345f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/hooks/python.md @@ -125,6 +125,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o | `ctx.hook_index` | Zero-based position in the authored hook list | | `ctx.run_id` | Current run ID | | `ctx.target` | Active target | +| `ctx.target_config` | The active target's database and schemas ([Target settings](../../python-nodes/overview.md#target-settings)) | | `ctx.vars` | Effective project variables | | `ctx.destination` | Destination relation metadata | | `ctx.relation(ref)` | Adapter-qualified relation for a [declared read](#declared-reads) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index 9d4aca5f7e..94e291ab16 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -254,6 +254,7 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `current_cursor_value` | `object \| None` | Current `MAX(cursor_column)` from the destination, or `None` if the table does not exist or has no cursor column | | `run_id` | `str` | Unique identifier for this execution run | | `target` | `str \| None` | Active target name (e.g. `dev`, `prod`) | +| `target_config` | `TargetSettings` | The active target's database, schema and loader schema ([Target settings](overview.md#target-settings)) | | `vars` | `dict` | Project variables (merged from project, target, and local config) | | `is_reload` | `bool` | `True` when `--reload` was passed | | `start_cursor_ts` | `datetime \| None` | Timestamp cursor start override from `--start-cursor-ts` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 1488233602..08b2cd3210 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -113,6 +113,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset |--------|-------------| | `ctx.run_id` | Unique identifier for this run | | `ctx.target` | Active target name | +| `ctx.target_config` | The active target's database and schemas, as SQLBuild resolves them (see [Target settings](#target-settings)) | | `ctx.vars` | Project variables | | `ctx.is_reload` | `True` when `--reload` was passed | | `ctx.adapter` / `ctx.connection` | Adapter and live connection | @@ -128,6 +129,30 @@ Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check context Providers can also be injected directly as function parameters by name. See [Providers](providers.md) for details. +### Target settings + +Read the active target's locations from `ctx.target_config` instead of keeping each target's schema names in Python. The values are resolved the way SQLBuild resolves them for models and loaders, including `sqlbuild_local.toml` overrides, and are read-only: + +| Field | Value | +|-------|-------| +| `name` | Active target name, or `None` without targets | +| `database` | Database models build into, or `None` when the adapter has none or the value is `preserve` | +| `schema` | Schema models build into, or `None` when it is `preserve` | +| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema` | +| `database_preserved` / `schema_preserved` | `True` when the target sets that value to `preserve` | + +```python +@asset +def partner_export(ctx): + """Export orders to the partner staging table.""" + target = ctx.target_config + staging = f"{target.database}.{target.loader_schema}.partner_staging" + ctx.execute_sql(f"CREATE OR REPLACE TABLE {staging} AS SELECT * FROM orders") + return ctx.result() +``` + +Connection settings and secrets are not part of `ctx.target_config`. + ## Returns and skips Tasks and assets return through `ctx.result(...)`: diff --git a/website/src/content/docs/docs/concepts/models/hooks/python.mdx b/website/src/content/docs/docs/concepts/models/hooks/python.mdx index 9bac6070ee..009f5e7f13 100644 --- a/website/src/content/docs/docs/concepts/models/hooks/python.mdx +++ b/website/src/content/docs/docs/concepts/models/hooks/python.mdx @@ -113,6 +113,7 @@ Python hooks declare a `HookContext` parameter named `ctx`, `context`, `_ctx`, o | `ctx.hook_index` | Zero-based position in the authored hook list | | `ctx.run_id` | Current run ID | | `ctx.target` | Active target | +| `ctx.target_config` | The active target's database and schemas ([Target settings](/docs/concepts/python-nodes/overview#target-settings)) | | `ctx.vars` | Effective project variables | | `ctx.destination` | Destination relation metadata | | `ctx.relation(ref)` | Adapter-qualified relation for a [declared read](#declared-reads) | diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 02bb37969d..124ad222de 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -234,6 +234,7 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `current_cursor_value` | `object \| None` | Current `MAX(cursor_column)` from the destination, or `None` if the table does not exist or has no cursor column | | `run_id` | `str` | Unique identifier for this execution run | | `target` | `str \| None` | Active target name (e.g. `dev`, `prod`) | +| `target_config` | `TargetSettings` | The active target's database, schema and loader schema ([Target settings](/docs/concepts/python-nodes/overview#target-settings)) | | `vars` | `dict` | Project variables (merged from project, target, and local config) | | `is_reload` | `bool` | `True` when `--reload` was passed | | `start_cursor_ts` | `datetime \| None` | Timestamp cursor start override from `--start-cursor-ts` | diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 25068aab90..2ef12f10d4 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -97,6 +97,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset |--------|-------------| | `ctx.run_id` | Unique identifier for this run | | `ctx.target` | Active target name | +| `ctx.target_config` | The active target's database and schemas, as SQLBuild resolves them (see [Target settings](#target-settings)) | | `ctx.vars` | Project variables | | `ctx.is_reload` | `True` when `--reload` was passed | | `ctx.adapter` / `ctx.connection` | Adapter and live connection | @@ -112,6 +113,30 @@ Task and asset contexts add `ctx.result(...)` and `ctx.skip(...)`. Check context Providers can also be injected directly as function parameters by name. See [Providers](/docs/concepts/python-nodes/providers) for details. +### Target settings + +Read the active target's locations from `ctx.target_config` instead of keeping each target's schema names in Python. The values are resolved the way SQLBuild resolves them for models and loaders, including `sqlbuild_local.toml` overrides, and are read-only: + +| Field | Value | +|-------|-------| +| `name` | Active target name, or `None` without targets | +| `database` | Database models build into, or `None` when the adapter has none or the value is `preserve` | +| `schema` | Schema models build into, or `None` when it is `preserve` | +| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema` | +| `database_preserved` / `schema_preserved` | `True` when the target sets that value to `preserve` | + +```python +@asset +def partner_export(ctx): + """Export orders to the partner staging table.""" + target = ctx.target_config + staging = f"{target.database}.{target.loader_schema}.partner_staging" + ctx.execute_sql(f"CREATE OR REPLACE TABLE {staging} AS SELECT * FROM orders") + return ctx.result() +``` + +Connection settings and secrets are not part of `ctx.target_config`. + ## Returns and skips Tasks and assets return through `ctx.result(...)`: From e47b4d56eb905c0390a8b122b8b08a96ae026ea4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:38:40 +0000 Subject: [PATCH 08/15] refactor: make TargetSettings a native record resolved once per command TargetSettings is now a native record built from Rust values, so command structs hold no new Python objects. debug, diff and load share one effective_target_namespace that resolves the connection and adapter defaults, replacing the keyword-built pyfunction each repeated. --- .../src/entries/_helpers/executor.rs | 5 - crates/sqlbuild-host/src/entries/models.rs | 1 - .../_helpers/contract_records/constants.rs | 13 ++ .../records/_helpers/native_classes/ids.rs | 2 +- .../sqlbuild-python-base/src/records/types.rs | 2 +- .../_helpers/build_command/execution.rs | 6 +- .../_helpers/build_command/lifecycle.rs | 8 +- .../bindings/_helpers/check/check_command.rs | 12 +- .../src/bindings/_helpers/check/check_core.rs | 20 +-- .../compile_attachment/registration.rs | 4 - .../_helpers/compile_attachment/target.rs | 45 ++---- .../_helpers/compile_driver/targets.rs | 149 +++++++----------- .../_helpers/debug_command/run_debug.rs | 47 ++---- .../bindings/_helpers/diff_report/query.rs | 34 +--- .../bindings/_helpers/load/load_command.rs | 33 +++- fensu.toml | 6 +- src/sqlbuild/_native.pyi | 22 ++- src/sqlbuild/_native_host.py | 1 - src/sqlbuild/_native_host.pyi | 1 - src/sqlbuild/executor/build/models.py | 2 +- src/sqlbuild/executor/load/models.py | 3 +- src/sqlbuild/executor/python_nodes/models.py | 2 +- src/sqlbuild/executor/run/models.py | 2 +- src/sqlbuild/runtime/contracts/models.py | 12 -- .../cli/commands/main/build/test_build.py | 19 ++- 25 files changed, 193 insertions(+), 258 deletions(-) diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index d59f27d80d..17302822a0 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -624,11 +624,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, - HostEntry { - name: "TargetSettings", - kind: HostKind::Class, - edge: HostEdge::Executor, - }, HostEntry { name: "ExecutionResourceKind", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 1fee19c724..70f6e3a9ab 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -329,7 +329,6 @@ pub enum ExecutorHost { PruneResult, ComputeLogStream, ConnectionHooks, - TargetSettings, ExecutionResourceKind, ExecutionHistoryStorageError, IntegrityConflictError, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index d77c2bcff3..a76c70eb70 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -150,6 +150,18 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), ], }; +const R_TARGET_SETTINGS: RecordSpec = RecordSpec { + name: "TargetSettings", + doc: "The active target's database and schemas, resolved as SQLBuild uses them.", + fields: &[ + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("loader_schema", FieldDefault::None), + FieldSpec::new("database_preserved", FieldDefault::False), + FieldSpec::new("schema_preserved", FieldDefault::False), + ], +}; const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { name: "LocalTargetConfig", doc: "Local developer overrides for one named target.", @@ -729,6 +741,7 @@ pub(crate) const CONTRACT_RECORDS: &[RecordSpec] = &[ R_TARGET_DIFF_CONFIG, R_TARGET_WAREHOUSES_CONFIG, R_TARGET_CONFIG, + R_TARGET_SETTINGS, R_LOCAL_TARGET_CONFIG, R_SETTINGS_CONFIG, R_SCOPES_CONFIG, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs index 36b3d857ae..69565744d3 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs @@ -567,6 +567,7 @@ native_id_names! { RecordId::TableTypeSource => "TableTypeSource", RecordId::TargetConfig => "TargetConfig", RecordId::TargetDiffConfig => "TargetDiffConfig", + RecordId::TargetSettings => "TargetSettings", RecordId::TargetWarehousesConfig => "TargetWarehousesConfig", RecordId::TestCommandRequest => "TestCommandRequest", RecordId::TestInvocation => "TestInvocation", @@ -689,7 +690,6 @@ native_id_names! { NativeAttr::BuildEffectiveRuntimeConfig => "build_effective_runtime_config", NativeAttr::BuildEffectiveSettingsConfig => "build_effective_settings_config", NativeAttr::BuildEffectiveTargetConfig => "build_effective_target_config", - NativeAttr::BuildEffectiveTargetNamespace => "build_effective_target_namespace", NativeAttr::BuildExecutionPlan => "build_execution_plan", NativeAttr::BuildFailedSqlbuildModelNames => "build_failed_sqlbuild_model_names", NativeAttr::BuildIdentity => "BUILD_IDENTITY", diff --git a/crates/sqlbuild-python-base/src/records/types.rs b/crates/sqlbuild-python-base/src/records/types.rs index 7d9043d0cb..ef1b92a365 100644 --- a/crates/sqlbuild-python-base/src/records/types.rs +++ b/crates/sqlbuild-python-base/src/records/types.rs @@ -560,6 +560,7 @@ pub enum RecordId { TableTypeSource, TargetConfig, TargetDiffConfig, + TargetSettings, TargetWarehousesConfig, TestCommandRequest, TestInvocation, @@ -682,7 +683,6 @@ pub enum NativeAttr { BuildEffectiveRuntimeConfig, BuildEffectiveSettingsConfig, BuildEffectiveTargetConfig, - BuildEffectiveTargetNamespace, BuildExecutionPlan, BuildFailedSqlbuildModelNames, BuildIdentity, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index d827946302..77ac47a567 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -364,7 +364,8 @@ pub(crate) fn execute_build_plan<'py>( &context.get("discovered_inputs")?, &project, (&connection_config, &adapter), - )?, + )? + .record(py)?, ), ("start_cursor_ts", start_ts.clone()), ("end_cursor_ts", end_ts.clone()), @@ -649,7 +650,8 @@ fn check_runtime<'py>( &context.get("discovered_inputs")?, &project, (&context.get("connection_config")?, &adapter), - )?, + )? + .record(py)?, ), ( "default_database", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 58fde65b3b..0230cc4f6f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -14,7 +14,9 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; -use crate::bindings::_helpers::compile_driver::targets::project_target_settings; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, project_target_settings, +}; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::planner_scope::loaders::build_intermediate_source_map; @@ -260,7 +262,7 @@ struct RuntimeParts<'a, 'py> { project: &'a Bound<'py, PyAny>, reload: bool, defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), - target_config: Bound<'py, PyAny>, + target_config: TargetSettingsValues, relations: (Bound<'py, PyAny>, Bound<'py, PyAny>), inputs: &'a LifecycleInputs<'py>, } @@ -284,7 +286,7 @@ fn node_runtime<'py>( ), ("vars", native_field(parts.project, "effective_vars")?), ("is_reload", boolean(py, parts.reload)), - ("target_config", parts.target_config.clone()), + ("target_config", parts.target_config.record(py)?), ("default_database", parts.defaults.0.clone()), ("default_schema", parts.defaults.1.clone()), ("start_cursor_ts", start_ts.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 9731d0d303..a3332b007c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -7,7 +7,9 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pi use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::compile_driver::targets::project_target_settings; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, project_target_settings, +}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; @@ -71,7 +73,7 @@ struct Prepared<'py> { relation_targets: Bound<'py, PyAny>, relation_refs: Bound<'py, PyAny>, defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), - target_config: Bound<'py, PyAny>, + target_config: TargetSettingsValues, project_relations: Bound<'py, PyAny>, } @@ -356,7 +358,7 @@ fn prepare<'py>( )?), )?, )?; - let target_config: Bound<'py, PyAny> = project_target_settings( + let target_config: TargetSettingsValues = project_target_settings( &discovered, &project, (&native_field(invocation, "connection_config")?, &adapter), @@ -413,7 +415,7 @@ fn node_runtime<'py>( ("target", native_field(&project, "effective_target_name")?), ("vars", native_field(&project, "effective_vars")?), ("is_reload", PyBool::new(py, false).to_owned().into_any()), - ("target_config", prepared.target_config.clone()), + ("target_config", prepared.target_config.record(py)?), ("default_database", prepared.defaults.0.clone()), ("default_schema", prepared.defaults.1.clone()), ("relation_targets", prepared.relation_targets.clone()), @@ -502,7 +504,7 @@ fn run_checks<'py>( validation_refs: prepared.relation_refs.clone(), project_relations: prepared.project_relations.clone(), providers: providers.clone(), - target_config: prepared.target_config.clone(), + target_config: Some(prepared.target_config.clone()), })?; record_run_state( invocation, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 6cbfd3a328..3e43c52fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -1,5 +1,6 @@ //! Python check planning: run-state recording, dependency relations and read-side preflight. +use crate::bindings::_helpers::compile_driver::targets::TargetSettingsValues; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; @@ -38,12 +39,7 @@ const READ_SIDE_OPTIONS: (&[&str], &[&str]) = ( "lifecycle_plan", "relation_targets", ], - &[ - "validation_refs", - "project_relations", - "providers", - "target_config", - ], + &["validation_refs", "project_relations", "providers"], ); fn user_error(py: Python<'_>, message: String) -> PyErr { @@ -268,7 +264,7 @@ pub(crate) struct ReadSideInputs<'py> { pub(crate) validation_refs: Bound<'py, PyAny>, pub(crate) project_relations: Bound<'py, PyAny>, pub(crate) providers: Bound<'py, PyAny>, - pub(crate) target_config: Bound<'py, PyAny>, + pub(crate) target_config: Option, } fn reference_kind(reference: &Bound<'_, PyAny>) -> PyResult { @@ -460,7 +456,13 @@ fn node_runtime<'py>(inputs: &ReadSideInputs<'py>) -> PyResult ("target", native_field(&project, "effective_target_name")?), ("vars", native_field(&project, "effective_vars")?), ("is_reload", false.into_pyobject(py)?.to_owned().into_any()), - ("target_config", inputs.target_config.clone()), + ( + "target_config", + match &inputs.target_config { + Some(settings) => settings.record(py)?, + None => py.None().into_bound(py), + }, + ), ( "default_database", inputs.adapter.call_method0("default_database")?, @@ -548,7 +550,7 @@ fn run_check_read_side_dependencies<'py>( validation_refs: arguments.value("validation_refs")?, project_relations: arguments.value("project_relations")?, providers: arguments.value("providers")?, - target_config: arguments.value("target_config")?, + target_config: None, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index a6aa6e901d..8ba218f583 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -35,10 +35,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { target::build_compile_target_context, module )?)?; - module.add_function(wrap_pyfunction!( - target::build_effective_target_namespace, - module - )?)?; module.add_function(wrap_pyfunction!( declaration_scope::build_declaration_scope, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 7bda60a51e..408b1c8897 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -13,8 +13,8 @@ use crate::bindings::_helpers::frontier::constants::{ }; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::compile_attachment::main::project_vars::project_vars; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; @@ -23,6 +23,8 @@ use sqlbuild_python_front::model_config::main::config::run_id::run_id; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; +/// Adapters whose database is a file, so a connection `database` is not a target location. +pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; /// The selected target's name and merged config, or `(None, None)`. pub(crate) fn resolved_target<'py>( @@ -150,35 +152,20 @@ pub(crate) fn connection_is_default_target(discovered: &Bound<'_, PyAny>) -> PyR Ok(true) } -/// Resolve the target and its effective database/schema without compiling resources. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_effective_target_namespace<'py>( - py: Python<'py>, - options: Option>, +/// The selected target's name, record, database and schema, resolved from `connection` and the adapter's defaults without compiling resources. +pub(crate) fn effective_target_namespace<'py>( + (discovered_inputs, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter_name, connection, adapter): (&str, &Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_effective_target_namespace", - options, - ( - &[ - "discovered_inputs", - "selected_target", - "connection_database", - "connection_schema", - "default_database", - "default_schema", - ], - &[], - ), - )?; - let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; - let selected_target: &Bound<'py, PyAny> = &arguments.value("selected_target")?; - let connection_database: Bound<'py, PyAny> = arguments.value("connection_database")?; - let connection_schema: Bound<'py, PyAny> = arguments.value("connection_schema")?; - let default_database: Bound<'py, PyAny> = arguments.value("default_database")?; - let default_schema: Bound<'py, PyAny> = arguments.value("default_schema")?; + let py: Python<'py> = adapter.py(); + let connection_database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name) { + py.None().into_bound(py) + } else { + mapping_get(connection, DATABASE_KEY)? + }; + let connection_schema: Bound<'py, PyAny> = mapping_get(connection, SCHEMA_KEY)?; + let default_database: Bound<'py, PyAny> = adapter.call_method0("default_database")?; + let default_schema: Bound<'py, PyAny> = adapter.call_method0("default_schema")?; let (target_name, target_config) = resolved_target(discovered_inputs, selected_target)?; let effective_vars: Bound<'py, PyAny> = project_vars(discovered_inputs, &target_config)?; let target_value = |key: &str| -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 37c2c0e772..648921f4b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -5,7 +5,7 @@ use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::target_checks::constants::{ @@ -41,17 +41,12 @@ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databrick const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; -use crate::bindings::_helpers::compile_attachment::target::{ - build_effective_target_namespace, resolved_target, -}; -use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; +use crate::bindings::_helpers::compile_attachment::target::resolved_target; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; -use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -418,16 +413,54 @@ fn managed_source_namespace<'py>( Ok((part("database")?, part("schema")?)) } -/// The active target as `TargetSettings`: `database` and `schema` as models resolve them, `None` with a flag when `preserve`, and the schema a managed source without its own loads into. +/// The active target's locations as SQLBuild resolves them; `preserve` stays as written until `record`. +#[derive(Clone)] +pub(crate) struct TargetSettingsValues { + name: Option, + database: Option, + schema: Option, + loader_schema: Option, +} + +impl TargetSettingsValues { + /// The `TargetSettings` record: a `preserve` location reads `None` with its flag set. + pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { + let located = |value: &Option| -> PyResult> { + value + .as_deref() + .filter(|value| *value != PRESERVE_TARGET_VALUE) + .into_bound_py_any(py) + }; + let preserved = |value: &Option| -> Bound<'py, PyAny> { + PyBool::new(py, value.as_deref() == Some(PRESERVE_TARGET_VALUE)) + .to_owned() + .into_any() + }; + new_record_with( + py, + RecordId::TargetSettings, + &[ + ("name", self.name.clone().into_bound_py_any(py)?), + ("database", located(&self.database)?), + ("schema", located(&self.schema)?), + ("loader_schema", located(&self.loader_schema)?), + ("database_preserved", preserved(&self.database)), + ("schema_preserved", preserved(&self.schema)), + ], + ) + } +} + +/// The target `target_name` with its resolved `database` and `schema`, and the schema a managed source without its own loads into. pub(crate) fn target_settings<'py>( (discovered, target_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), + (database, schema): (Option, Option), (connection, effective_vars, adapter): ( &Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>, ), -) -> PyResult> { +) -> PyResult { let py: Python<'py> = adapter.py(); let target_config: Bound<'py, PyAny> = if target_name.is_none() { new_record(py, RecordId::TargetConfig, (), None)? @@ -441,50 +474,16 @@ pub(crate) fn target_settings<'py>( }; let none: Bound<'py, PyAny> = py.None().into_bound(py); let (_, loader_schema): Namespace = managed_source_namespace( - (none.clone(), none.clone()), + (none.clone(), none), (&target_config, connection, effective_vars), adapter, )?; - let preserved = |value: &Bound<'py, PyAny>| -> PyResult { - Ok(text_or_none(value)?.as_deref() == Some(PRESERVE_TARGET_VALUE)) - }; - let (database_preserved, schema_preserved) = (preserved(&database)?, preserved(&schema)?); - call_with( - &host_entry(py, ExecutorHost::TargetSettings)?, - &[ - ("name", target_name.clone()), - ( - "database", - if database_preserved { - none.clone() - } else { - database - }, - ), - ( - "schema", - if schema_preserved { - none.clone() - } else { - schema - }, - ), - ( - "loader_schema", - loader_schema - .filter(|value| value != PRESERVE_TARGET_VALUE) - .into_bound_py_any(py)?, - ), - ( - "database_preserved", - PyBool::new(py, database_preserved).to_owned().into_any(), - ), - ( - "schema_preserved", - PyBool::new(py, schema_preserved).to_owned().into_any(), - ), - ], - ) + Ok(TargetSettingsValues { + name: target_name.extract()?, + database, + schema, + loader_schema, + }) } /// `target_settings` of a compiled project's target. @@ -492,12 +491,12 @@ pub(crate) fn project_target_settings<'py>( discovered: &Bound<'py, PyAny>, project: &Bound<'py, PyAny>, (connection, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { +) -> PyResult { target_settings( (discovered, &native_field(project, "effective_target_name")?), ( - native_field(project, "effective_target_database")?, - native_field(project, "effective_target_schema")?, + native_field(project, "effective_target_database")?.extract()?, + native_field(project, "effective_target_schema")?.extract()?, ), ( connection, @@ -507,46 +506,6 @@ pub(crate) fn project_target_settings<'py>( ) } -/// `target_settings` of `selected_target` for a command that does not compile, such as `load`. -pub(crate) fn selected_target_settings<'py>( - (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - adapter_name: &str, - (connection, effective_vars, adapter): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), -) -> PyResult> { - let py: Python<'py> = adapter.py(); - let connection_database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name) { - py.None().into_bound(py) - } else { - mapping_get(connection, "database")? - }; - let namespace: Bound<'py, PyTuple> = build_effective_target_namespace( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", selected_target.clone()), - ("connection_database", connection_database), - ("connection_schema", mapping_get(connection, "schema")?), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ("default_schema", adapter.call_method0("default_schema")?), - ], - )?), - )?; - target_settings( - (discovered, &namespace.get_item(0)?), - (namespace.get_item(2)?, namespace.get_item(3)?), - (connection, effective_vars, adapter), - ) -} - /// A managed source's `database` and `schema` with project vars expanded and no context templates. pub(crate) fn expanded_loader_namespace<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 5be09d62b7..9abb15a730 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -33,11 +33,9 @@ use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehou use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; -use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; -use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; +use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; @@ -308,41 +306,14 @@ fn build_report( })?; let connection_config: Bound<'_, PyDict> = effective_connection(&context, (&adapter_name, &base_connection))?.cast_into()?; - let namespace_adapter: String = adapter_name.str()?.extract()?; - let namespaceless: bool = NAMESPACELESS_ADAPTERS.contains(&namespace_adapter.as_str()); - let namespace: Bound<'_, PyTuple> = build_effective_target_namespace( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", context.discovered_inputs.clone()), - ("selected_target", context.selected_target.clone()), - ( - "connection_database", - connection_config - .get_item("database")? - .filter(|_| !namespaceless) - .unwrap_or_else(|| py.None().into_bound(py)), - ), - ( - "connection_schema", - connection_config - .get_item("schema")? - .unwrap_or_else(|| py.None().into_bound(py)), - ), - ( - "default_database", - context.adapter.call_method0("default_database")?, - ), - ( - "default_schema", - context.adapter.call_method0("default_schema")?, - ), - ], - )?), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .cast_into()?; + let namespace: Bound<'_, PyTuple> = effective_target_namespace( + (&context.discovered_inputs, &context.selected_target), + ( + &adapter_name.str()?.extract::()?, + connection_config.as_any(), + &context.adapter, + ), + )?; let target_config: Bound<'_, PyAny> = namespace.get_item(1)?; let connection_name: Option = if target_config.is_none() { None diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 29afbc7e3f..fda90eed6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -16,9 +16,9 @@ use sqlbuild_cli::diff_report::types::PhaseSeconds; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace; +use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use crate::bindings::_helpers::diff_report::support::{ - call, cli_error, error_text, none, object, progress_line, + call, cli_error, error_text, object, progress_line, }; use crate::bindings::_helpers::diff_report::tolerances::cli_tolerance_overrides; use sqlbuild_analysis::diff_selection::main::raw_query_shape::raw_query_shape; @@ -30,7 +30,6 @@ use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cl use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; @@ -38,7 +37,6 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_con use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; const GENERIC_DIALECT: &str = "generic"; -pub(crate) const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; fn failure(py: Python<'_>, message: String, code: &str, help: Option<&str>) -> PyErr { cli_error( @@ -197,29 +195,11 @@ pub(crate) fn prepare_query_diff<'py>( ], )?), )?; - let database: Bound<'py, PyAny> = if NAMESPACELESS_ADAPTERS.contains(&adapter_name.as_str()) { - none(py) - } else { - mapping_get(&config, "database")? - }; - let namespace: Bound<'py, PyAny> = build_effective_target_namespace( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", selected_target.clone()), - ("connection_database", database.clone()), - ("connection_schema", mapping_get(&config, "schema")?), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ("default_schema", adapter.call_method0("default_schema")?), - ], - )?), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + let namespace: Bound<'py, PyAny> = effective_target_namespace( + (discovered, &selected_target), + (&adapter_name, &config, &adapter), + )? + .into_any(); let ResolvedNamespace { target_name, database, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index b1c88db5d5..b44b218eb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -21,8 +21,9 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; +use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::targets::selected_target_settings; +use crate::bindings::_helpers::compile_driver::targets::{TargetSettingsValues, target_settings}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -80,7 +81,31 @@ struct Preparation<'py> { cursor_overrides: Bound<'py, PyAny>, provider_session: Bound<'py, PyAny>, project_relations: Bound<'py, PyAny>, - target_config: Bound<'py, PyAny>, + target_config: TargetSettingsValues, +} + +/// The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. +fn selected_target_settings<'py>( + (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter_name: &str, + (connection, effective_vars, adapter): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult { + let namespace: Bound<'py, PyTuple> = effective_target_namespace( + (discovered, selected_target), + (adapter_name, connection, adapter), + )?; + target_settings( + (discovered, &namespace.get_item(0)?), + ( + namespace.get_item(2)?.extract()?, + namespace.get_item(3)?.extract()?, + ), + (connection, effective_vars, adapter), + ) } fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { @@ -363,7 +388,7 @@ fn prepare<'py>( native_field(discovered, "providers")?, )], )?; - let target_config: Bound<'py, PyAny> = selected_target_settings( + let target_config: TargetSettingsValues = selected_target_settings( (discovered, &native_field(request, "selected_target")?), &adapter_name.extract::()?, (&connection_config, &runtime.1, &adapter), @@ -403,7 +428,7 @@ fn runtime_params<'py>( ("target", target_name.clone()), ("vars", effective_vars.clone()), ("is_reload", native_field(request, "reload")?), - ("target_config", preparation.target_config.clone()), + ("target_config", preparation.target_config.record(py)?), ( "start_cursor_ts", cursor_timestamp(&native_field(overrides, "start_ts")?)?, diff --git a/fensu.toml b/fensu.toml index 8e8fb2c0f1..65b77a6258 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2209,7 +2209,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs=30", @@ -2312,11 +2312,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/output.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=19", + "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index da45f67a47..4c96aba346 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -131,7 +131,7 @@ from sqlbuild.runtime.compute_logs.models import ( PruneResult, ) from sqlbuild.runtime.compute_logs.types import ComputeLogStream -from sqlbuild.runtime.contracts.models import ConnectionHooks, TargetSettings +from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.execution_history.models import ( EventFilter, @@ -2281,7 +2281,6 @@ def run_check_read_side_dependencies( validation_refs: frozenset[SqlResourceRef] | None = None, project_relations: dict[SqlResourceRef, str] | None = None, providers: ProviderContainer | None = None, - target_config: TargetSettings | None = None, ) -> tuple[PythonNodeExecutionResult, ...]: ... def finalize_build_cost(finalization: BuildCostFinalization) -> CostRunRecord | None: ... def format_cost_breakdown( @@ -11901,6 +11900,25 @@ class TargetDiffConfig(NativeRecord): max_full_rows: int | Literal["unlimited"] | None = ..., ) -> None: ... +class TargetSettings(NativeRecord): + """The active target's database and schemas, resolved as SQLBuild uses them.""" + + name: str | None + database: str | None + schema: str | None + loader_schema: str | None + database_preserved: bool + schema_preserved: bool + def __init__( + self, + name: str | None = ..., + database: str | None = ..., + schema: str | None = ..., + loader_schema: str | None = ..., + database_preserved: bool = ..., + schema_preserved: bool = ..., + ) -> None: ... + class TargetWarehousesConfig(NativeRecord): """Optional per-command-group default warehouses for one target.""" diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 11b8f0ec9e..46a0ef8207 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -554,7 +554,6 @@ "PruneResult": ("sqlbuild.runtime.compute_logs.models", "PruneResult"), "ComputeLogStream": ("sqlbuild.runtime.compute_logs.types", "ComputeLogStream"), "ConnectionHooks": ("sqlbuild.runtime.contracts.models", "ConnectionHooks"), - "TargetSettings": ("sqlbuild.runtime.contracts.models", "TargetSettings"), "ExecutionResourceKind": ("sqlbuild.runtime.contracts.types", "ExecutionResourceKind"), "ExecutionHistoryStorageError": ( "sqlbuild.runtime.execution_history.exceptions", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 34fc9f956d..c3d24a82bd 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -518,7 +518,6 @@ from sqlbuild.runtime.compute_logs.models import FinalCaptureMetadata as FinalCa from sqlbuild.runtime.compute_logs.models import PruneResult as PruneResult from sqlbuild.runtime.compute_logs.types import ComputeLogStream as ComputeLogStream from sqlbuild.runtime.contracts.models import ConnectionHooks as ConnectionHooks -from sqlbuild.runtime.contracts.models import TargetSettings as TargetSettings from sqlbuild.runtime.contracts.types import ExecutionResourceKind as ExecutionResourceKind from sqlbuild.runtime.event_exporting.exceptions import ( EventExporterInputError as EventExporterInputError, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index cd181d2bdc..7c147d5386 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -18,6 +18,7 @@ SnapshotsConfig, SourceLoadPlanEntry, SqlTestPlanEntry, + TargetSettings, ) from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode @@ -34,7 +35,6 @@ from sqlbuild.microbatches.types import MicrobatchEventStore from sqlbuild.provider.main.runtime import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.runtime.contracts.models import TargetSettings from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, NodeStartCallback from sqlbuild.runtime.execution_limits.models import RunItemTimeouts diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index ec4ef6be0b..38f6e970b0 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry, TargetSettings from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent @@ -27,7 +27,6 @@ from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.runtime.contracts.models import TargetSettings from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, ExecutionResourceKind diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 1c794faf75..661af2edc7 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -8,6 +8,7 @@ from datetime import datetime from typing import Any +from sqlbuild._native import TargetSettings from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( @@ -30,7 +31,6 @@ from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.runtime.contracts.models import TargetSettings from sqlbuild.runtime.contracts.types import NodeStartCallback diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 9177b7d6b5..674a5fba62 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -26,6 +26,7 @@ PythonHookEntry, SourceEntry, StartCursorsConfig, + TargetSettings, TimestampValue, ) from sqlbuild._native import cursor_compare as compare @@ -56,7 +57,6 @@ from sqlbuild.microbatches.types import MicrobatchEventStore from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.runtime.contracts.models import TargetSettings @dataclass(frozen=True, kw_only=True, init=False) diff --git a/src/sqlbuild/runtime/contracts/models.py b/src/sqlbuild/runtime/contracts/models.py index e0005f48cd..ed2c0f8d6a 100644 --- a/src/sqlbuild/runtime/contracts/models.py +++ b/src/sqlbuild/runtime/contracts/models.py @@ -8,18 +8,6 @@ from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback -@dataclass(frozen=True) -class TargetSettings: - """The active target's database and schemas, resolved as SQLBuild uses them.""" - - name: str | None - database: str | None - schema: str | None - loader_schema: str | None - database_preserved: bool = False - schema_preserved: bool = False - - @dataclass(frozen=True) class ConnectionHooks: """Progress and connection lifecycle callbacks for long-running operations.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 7bd24c0417..0e11ed812d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -37,6 +37,15 @@ table_exists, ) +_TARGET_CONFIG_RECORDER: str = ( + "import dataclasses, json\n" + "from pathlib import Path\n\n" + "def record(ctx, node):\n" + " line = {'node': node, **dataclasses.asdict(ctx.target_config)}\n" + " with Path(__file__).parents[1].joinpath('target_config.jsonl').open('a') as out:\n" + " out.write(json.dumps(line) + '\\n')\n" +) + @pytest.mark.parametrize( "test_case", @@ -1809,16 +1818,6 @@ def test_given_loader_status_result_when_building_then_persists_loader_status_ro ) -_TARGET_CONFIG_RECORDER: str = ( - "import dataclasses, json\n" - "from pathlib import Path\n\n" - "def record(ctx, node):\n" - " line = {'node': node, **dataclasses.asdict(ctx.target_config)}\n" - " with Path(__file__).parents[1].joinpath('target_config.jsonl').open('a') as out:\n" - " out.write(json.dumps(line) + '\\n')\n" -) - - @pytest.mark.parametrize( "test_case", [ From 78d4c1cf58b58ebad54447d5aa5e2f83cc0084f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:15:22 +0000 Subject: [PATCH 09/15] feat: report statement counts and compile time after sqb test and scenario test --- .../_helpers/history_function.rs | 14 ++ .../src/cost_allocation/_helpers/mod.rs | 1 + .../src/cost_allocation/main/mod.rs | 1 + .../cost_allocation/main/query_history_sql.rs | 12 +- .../main/statement_timings_sql.rs | 31 ++++ .../src/cost_allocation/tests.rs | 2 + .../tests/test_statement_timings.rs | 21 +++ .../src/progress/_helpers/statements.rs | 11 ++ .../sqlbuild-core/src/progress/constants.rs | 3 + .../progress/main/consume_progress_event.rs | 3 +- .../progress/main/format_statement_summary.rs | 45 +++++ crates/sqlbuild-core/src/progress/main/mod.rs | 2 + .../progress/main/statement_summary_json.rs | 30 ++++ crates/sqlbuild-core/src/progress/models.rs | 35 +++- .../src/progress/tests/test_progress.rs | 159 +++++++++++++++++- .../src/build_progress/models.rs | 6 + .../bindings/_helpers/result_output/mod.rs | 1 + .../_helpers/result_output/statements.rs | 136 +++++++++++++++ .../_helpers/scenario_command/test_command.rs | 19 ++- .../_helpers/sql_test_command/test_command.rs | 24 ++- .../sqlbuild/references/docs/cli/scenario.md | 4 +- .../sqlbuild/references/docs/cli/test.md | 25 +++ .../cli/commands/main/test/_test_types.py | 11 ++ .../main/test/test_statement_summary.py | 126 ++++++++++++++ .../src/content/docs/docs/cli/scenario.mdx | 4 +- website/src/content/docs/docs/cli/test.mdx | 24 +++ 26 files changed, 731 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-core/src/cost_allocation/_helpers/history_function.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs create mode 100644 crates/sqlbuild-core/src/progress/main/format_statement_summary.rs create mode 100644 crates/sqlbuild-core/src/progress/main/statement_summary_json.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/history_function.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/history_function.rs new file mode 100644 index 0000000000..9cb38a0a0d --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/history_function.rs @@ -0,0 +1,14 @@ +//! The Snowflake `QUERY_HISTORY` table function, qualified by database when there is one. + +use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION; + +/// `INFORMATION_SCHEMA.QUERY_HISTORY`, under `database` when it is given. +pub(crate) fn query_history_function(database: Option<&str>) -> String { + match database { + None => QUERY_HISTORY_FUNCTION.to_owned(), + Some(database) => format!( + "\"{}\".{QUERY_HISTORY_FUNCTION}", + database.to_uppercase().replace('"', "\"\"") + ), + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs index a9f8414daa..309c918ce6 100644 --- a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod allocation; pub(crate) mod decimal; +pub(crate) mod history_function; pub(crate) mod query_text; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/mod.rs b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs index d4b3364262..b0a502a247 100644 --- a/crates/sqlbuild-core/src/cost_allocation/main/mod.rs +++ b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs @@ -1,2 +1,3 @@ pub mod allocate_run_cost; pub mod query_history_sql; +pub mod statement_timings_sql; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs index 54fa520146..16b0b1ae7f 100644 --- a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs +++ b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs @@ -1,19 +1,9 @@ //! `query_history_sql`: the Snowflake `QUERY_HISTORY` read of one end-time window. +use crate::cost_allocation::_helpers::history_function::query_history_function; use crate::cost_allocation::_helpers::query_text::query_text_expression; -use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION; use crate::cost_allocation::models::QueryHistoryWindow; -fn query_history_function(database: Option<&str>) -> String { - match database { - None => QUERY_HISTORY_FUNCTION.to_owned(), - Some(database) => format!( - "\"{}\".{QUERY_HISTORY_FUNCTION}", - database.to_uppercase().replace('"', "\"\"") - ), - } -} - /// Query-history SQL for queries ending after `start`, and before `end` when it is bounded. pub fn query_history_sql(window: &QueryHistoryWindow<'_>) -> String { let end_range: String = match window.end { diff --git a/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs new file mode 100644 index 0000000000..af2525c090 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs @@ -0,0 +1,31 @@ +//! `statement_timings_sql`: compile and execution time of known Snowflake queries. + +use crate::cost_allocation::_helpers::history_function::query_history_function; +use crate::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; + +/// One query-history read summing compile and execution milliseconds of `query_ids`, among +/// queries that ended in the last `lookback_seconds`. +#[must_use] +pub fn statement_timings_sql( + database: Option<&str>, + query_ids: &[String], + lookback_seconds: u64, +) -> String { + let identifiers: String = query_ids + .iter() + .map(|query_id| format!("'{}'", query_id.replace('\'', "''"))) + .collect::>() + .join(", "); + format!( + "SELECT + COUNT(*), + COALESCE(SUM(COMPILATION_TIME), 0), + COALESCE(SUM(EXECUTION_TIME), 0) +FROM TABLE({}( + END_TIME_RANGE_START => DATEADD('second', -{lookback_seconds}, CURRENT_TIMESTAMP()), + RESULT_LIMIT => {QUERY_HISTORY_RESULT_LIMIT} +)) +WHERE QUERY_ID IN ({identifiers})", + query_history_function(database), + ) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests.rs b/crates/sqlbuild-core/src/cost_allocation/tests.rs index 6949838ed8..a1d089ca93 100644 --- a/crates/sqlbuild-core/src/cost_allocation/tests.rs +++ b/crates/sqlbuild-core/src/cost_allocation/tests.rs @@ -4,5 +4,7 @@ mod allocation; mod decimal; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_statement_timings.rs"] +mod statement_timings; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs new file mode 100644 index 0000000000..ef03ff93a7 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs @@ -0,0 +1,21 @@ +use crate::cost_allocation::main::statement_timings_sql::statement_timings_sql; + +#[test] +fn given_query_ids_when_rendering_timings_read_then_ids_are_quoted_and_database_qualified() { + let query_ids: Vec = vec!["01a-1".to_owned(), "it's".to_owned()]; + + let sql: String = statement_timings_sql(Some("analytics\"db"), &query_ids, 720); + + assert_eq!( + sql, + "SELECT + COUNT(*), + COALESCE(SUM(COMPILATION_TIME), 0), + COALESCE(SUM(EXECUTION_TIME), 0) +FROM TABLE(\"ANALYTICS\"\"DB\".INFORMATION_SCHEMA.QUERY_HISTORY( + END_TIME_RANGE_START => DATEADD('second', -720, CURRENT_TIMESTAMP()), + RESULT_LIMIT => 10000 +)) +WHERE QUERY_ID IN ('01a-1', 'it''s')" + ); +} diff --git a/crates/sqlbuild-core/src/progress/_helpers/statements.rs b/crates/sqlbuild-core/src/progress/_helpers/statements.rs index 76e95d91c5..f536c2cdce 100644 --- a/crates/sqlbuild-core/src/progress/_helpers/statements.rs +++ b/crates/sqlbuild-core/src/progress/_helpers/statements.rs @@ -142,3 +142,14 @@ pub(crate) fn consume_statement_terminal( projection.statement_query_ids.remove(statement_id); (projection, line) } + +/// Count a finished statement by kind, keeping its query identifier. +pub(crate) fn count_statement(projection: &mut ProgressProjection, event: &ProgressEvent) { + let query_id: Option = statement_query_id(projection, event); + let statements = &mut projection.statements; + *statements + .by_kind + .entry(event.payload.statement_kind.clone()) + .or_default() += 1; + statements.query_ids.extend(query_id); +} diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index bcdf9d0799..e8289a5288 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -127,3 +127,6 @@ pub(crate) const AUDIT_OUTCOME_STATUSES: [(&str, &str); 5] = [ ("insufficient", "INSUFFICIENT"), ("skipped", "SKIP"), ]; + +/// Where reported statement compile and execution time comes from. +pub(crate) const STATEMENT_TIMING_SOURCE: &str = "snowflake_query_history"; diff --git a/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs index 09c10fccd3..b2aaa9c108 100644 --- a/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs +++ b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs @@ -6,7 +6,7 @@ use crate::progress::_helpers::operations::{ use crate::progress::_helpers::resources::{consume_resource_start, consume_resource_terminal}; use crate::progress::_helpers::statements::{ consume_statement_start, consume_statement_submitted, consume_statement_terminal, - statement_heartbeat_line, + count_statement, statement_heartbeat_line, }; use crate::progress::constants::{ OPERATION_START, RESOURCE_START, RETRY_SCHEDULED, STATEMENT_COMPLETED, STATEMENT_FAILED, @@ -46,6 +46,7 @@ pub fn consume_progress_event( let line: Option = statement_heartbeat_line(&projection, event); (projection, line) } else if event_type == STATEMENT_COMPLETED || event_type == STATEMENT_FAILED { + count_statement(&mut projection, event); consume_statement_terminal(projection, event) } else { (projection, None) diff --git a/crates/sqlbuild-core/src/progress/main/format_statement_summary.rs b/crates/sqlbuild-core/src/progress/main/format_statement_summary.rs new file mode 100644 index 0000000000..c189b1de47 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/format_statement_summary.rs @@ -0,0 +1,45 @@ +//! The statement summary line of `sqb test` and `sqb scenario test`. + +use crate::progress::models::{StatementSummary, StatementTiming}; + +/// `Statements: 12 (SELECT 8, CREATE 2, DROP 2); compile 1.5s, execution 0.8s`; nothing without statements. +#[must_use] +pub fn format_statement_summary( + summary: &StatementSummary, + timing: &StatementTiming, +) -> Option { + let total: u64 = summary.by_kind.values().sum(); + if total == 0 { + return None; + } + let mut kinds: Vec<(&String, &u64)> = summary.by_kind.iter().collect(); + kinds.sort_by(|(left_kind, left), (right_kind, right)| { + right.cmp(left).then_with(|| left_kind.cmp(right_kind)) + }); + let kinds: String = kinds + .iter() + .map(|(kind, count)| format!("{kind} {count}")) + .collect::>() + .join(", "); + let time: String = match timing { + StatementTiming::Reported(timings) => { + let identified: u64 = u64::try_from(summary.query_ids.len()).unwrap_or(u64::MAX); + let coverage: String = if timings.timed_statements < identified { + format!( + " ({} of {identified} statements in query history)", + timings.timed_statements + ) + } else { + String::new() + }; + format!( + "compile {:.1}s, execution {:.1}s{coverage}", + timings.compile_ms / 1000.0, + timings.execution_ms / 1000.0 + ) + } + StatementTiming::NotReported(adapter) => format!("compile time not reported by {adapter}"), + StatementTiming::Failed(reason) => format!("compile time unavailable ({reason})"), + }; + Some(format!("Statements: {total} ({kinds}); {time}")) +} diff --git a/crates/sqlbuild-core/src/progress/main/mod.rs b/crates/sqlbuild-core/src/progress/main/mod.rs index e8f7c3c6f5..c912a2b982 100644 --- a/crates/sqlbuild-core/src/progress/main/mod.rs +++ b/crates/sqlbuild-core/src/progress/main/mod.rs @@ -8,8 +8,10 @@ pub mod format_execution_header; pub mod format_expectation_detail; pub mod format_expectation_name; pub mod format_inspection_record; +pub mod format_statement_summary; pub mod format_status_spinner_line; pub mod is_planning_completion_message; pub mod is_planning_failure_message; pub mod machine_output_mode; pub mod progress_terminal_duration; +pub mod statement_summary_json; diff --git a/crates/sqlbuild-core/src/progress/main/statement_summary_json.rs b/crates/sqlbuild-core/src/progress/main/statement_summary_json.rs new file mode 100644 index 0000000000..a5f7f02cc1 --- /dev/null +++ b/crates/sqlbuild-core/src/progress/main/statement_summary_json.rs @@ -0,0 +1,30 @@ +//! The `statements` section of the `sqb test` and `sqb scenario test` JSON documents. + +use serde_json::{Value, json}; + +use crate::progress::constants::STATEMENT_TIMING_SOURCE; +use crate::progress::models::{StatementSummary, StatementTiming}; + +/// `{total, by_kind, compile_time}`; `compile_time` is null when the adapter reports none. +#[must_use] +pub fn statement_summary_json(summary: &StatementSummary, timing: &StatementTiming) -> Value { + let compile_time: Value = match timing { + StatementTiming::Reported(timings) => json!({ + "source": STATEMENT_TIMING_SOURCE, + "compile_ms": timings.compile_ms, + "execution_ms": timings.execution_ms, + "timed_statements": timings.timed_statements, + "identified_statements": summary.query_ids.len(), + }), + StatementTiming::NotReported(_) => Value::Null, + StatementTiming::Failed(reason) => json!({ + "source": STATEMENT_TIMING_SOURCE, + "error": reason, + }), + }; + json!({ + "total": summary.by_kind.values().sum::(), + "by_kind": summary.by_kind, + "compile_time": compile_time, + }) +} diff --git a/crates/sqlbuild-core/src/progress/models.rs b/crates/sqlbuild-core/src/progress/models.rs index a2808e6b64..f9d0f1612a 100644 --- a/crates/sqlbuild-core/src/progress/models.rs +++ b/crates/sqlbuild-core/src/progress/models.rs @@ -1,6 +1,6 @@ //! The lifecycle facts progress projection reads, and the invocation projection state. -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use crate::progress::types::DifferenceSample; @@ -83,6 +83,33 @@ pub struct ProgressProjection { pub(crate) resource_names: HashMap, pub(crate) statement_starts: HashMap, pub(crate) statement_query_ids: HashMap, + pub(crate) statements: StatementSummary, +} + +/// The warehouse statements an invocation finished: how many of each leading keyword, and their query identifiers. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StatementSummary { + pub by_kind: BTreeMap, + pub query_ids: Vec, +} + +/// Warehouse-reported time of the statements found in query history. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct StatementTimings { + pub compile_ms: f64, + pub execution_ms: f64, + /// How many of the summary's query identifiers query history returned. + pub timed_statements: u64, +} + +/// What the warehouse reports about the time of an invocation's statements. +#[derive(Clone, Debug, PartialEq)] +pub enum StatementTiming { + Reported(StatementTimings), + /// The adapter, by name, keeps no per-statement compile time. + NotReported(String), + /// Reading query history failed, for this reason. + Failed(String), } impl ProgressProjection { @@ -116,6 +143,12 @@ impl ProgressProjection { .insert(resource_attempt_id.to_owned(), outcome.to_owned()); } + /// The warehouse statements finished so far. + #[must_use] + pub fn statement_summary(&self) -> &StatementSummary { + &self.statements + } + /// Whether a visible operation with this name currently owns progress. #[must_use] pub fn is_operation_active(&self, operation_name: &str) -> bool { diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress.rs b/crates/sqlbuild-core/src/progress/tests/test_progress.rs index 9ffb9e3b11..bfcca92e47 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress.rs @@ -1,4 +1,12 @@ -use crate::progress::models::{ProgressEvent, ProgressPayload, ProgressSettings}; +use std::collections::BTreeMap; + +use crate::progress::main::consume_progress_event::consume_progress_event; +use crate::progress::main::format_statement_summary::format_statement_summary; +use crate::progress::main::statement_summary_json::statement_summary_json; +use crate::progress::models::{ + ProgressEvent, ProgressPayload, ProgressProjection, ProgressSettings, StatementSummary, + StatementTiming, StatementTimings, +}; use crate::progress::tests::helpers::{ attempt_event, audit_rows, operation_event, orders_claim, plain, project_case, }; @@ -265,3 +273,152 @@ fn given_standalone_audit_when_projecting_then_row_names_target_and_outcome() { ); } } + +/// A statement event of `kind`, inside the `orders` attempt when `known` and outside any resource otherwise. +fn statement_event( + (event_id, event_type): (&str, &str), + (statement_id, kind, query_id): (&str, &str, Option<&str>), + known: bool, +) -> ProgressEvent { + let base: ProgressEvent = if known { + attempt_event(event_id, event_type, ("a1", None)) + } else { + ProgressEvent { + event_id: event_id.to_owned(), + event_type: event_type.to_owned(), + ..ProgressEvent::default() + } + }; + ProgressEvent { + statement_id: Some(statement_id.to_owned()), + payload: ProgressPayload { + statement_kind: kind.to_owned(), + query_id: query_id.map(str::to_owned), + ..base.payload + }, + ..base + } +} + +#[test] +fn given_statement_terminals_when_projecting_then_every_finished_statement_is_counted_once() { + let events: Vec = vec![ + attempt_event("r1", "resource_attempt_started", ("a1", None)), + statement_event(("s1", "statement_started"), ("st1", "CREATE", None), true), + statement_event(("s2", "statement_submitted"), ("st1", "CREATE", Some("q1")), true), + // The terminal omits the query identifier its submission reported. + statement_event(("s3", "statement_completed"), ("st1", "CREATE", None), true), + // A redelivered terminal is not counted again. + statement_event(("s3", "statement_completed"), ("st1", "CREATE", None), true), + // Statements outside any known resource, with and without a start, still count. + statement_event(("s4", "statement_completed"), ("st2", "WITH", Some("q2")), false), + statement_event(("s5", "statement_failed"), ("st3", "DROP", None), false), + statement_event(("s6", "statement_completed"), ("st4", "WITH", Some("q3")), false), + // Starts, submissions and heartbeats alone are not finished statements. + statement_event(("s7", "statement_started"), ("st5", "SELECT", None), false), + statement_event(("s8", "statement_heartbeat"), ("st5", "SELECT", Some("q5")), false), + ]; + let mut projection: ProgressProjection = ProgressProjection::new(plain()); + for event in &events { + projection = consume_progress_event(projection, event).0; + } + let summary: &StatementSummary = projection.statement_summary(); + assert_eq!( + summary.by_kind, + BTreeMap::from([ + ("CREATE".to_owned(), 1), + ("DROP".to_owned(), 1), + ("WITH".to_owned(), 2), + ]) + ); + assert_eq!(summary.query_ids, ["q1", "q2", "q3"]); +} + +#[test] +fn given_statement_summary_when_formatting_then_line_and_json_report_kinds_and_timing() { + let summary: StatementSummary = StatementSummary { + by_kind: BTreeMap::from([ + ("CREATE".to_owned(), 2), + ("DROP".to_owned(), 2), + ("SELECT".to_owned(), 7), + ("ALTER".to_owned(), 1), + ]), + query_ids: (0..12).map(|index| format!("q{index}")).collect(), + }; + let complete: StatementTiming = StatementTiming::Reported(StatementTimings { + compile_ms: 1540.0, + execution_ms: 260.0, + timed_statements: 12, + }); + let partial: StatementTiming = StatementTiming::Reported(StatementTimings { + compile_ms: 1000.0, + execution_ms: 0.0, + timed_statements: 10, + }); + let cases: [(&str, &StatementSummary, StatementTiming, Option<&str>); 5] = [ + ( + "kinds run from most to least frequent, ties by name", + &summary, + complete.clone(), + Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s", + ), + ), + ( + "statements missing from query history are called out", + &summary, + partial, + Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.0s, execution 0.0s (10 of 12 statements in query history)", + ), + ), + ( + "an adapter without query history says so", + &summary, + StatementTiming::NotReported("duckdb".to_owned()), + Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile time not reported by duckdb", + ), + ), + ( + "a failed read says why", + &summary, + StatementTiming::Failed("query history read failed".to_owned()), + Some( + "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile time unavailable (query history read failed)", + ), + ), + ( + "a run without statements has no line", + &StatementSummary::default(), + complete.clone(), + None, + ), + ]; + for (description, summary, timing, expected) in cases { + assert_eq!( + format_statement_summary(summary, &timing).as_deref(), + expected, + "{description}" + ); + } + assert_eq!( + statement_summary_json(&summary, &complete), + serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": { + "source": "snowflake_query_history", + "compile_ms": 1540.0, + "execution_ms": 260.0, + "timed_statements": 12, + "identified_statements": 12, + }, + }) + ); + assert_eq!( + statement_summary_json(&summary, &StatementTiming::NotReported("duckdb".to_owned())) + ["compile_time"], + serde_json::Value::Null + ); +} diff --git a/crates/sqlbuild-python-base/src/build_progress/models.rs b/crates/sqlbuild-python-base/src/build_progress/models.rs index dd2c38c3b9..36b5e2b04c 100644 --- a/crates/sqlbuild-python-base/src/build_progress/models.rs +++ b/crates/sqlbuild-python-base/src/build_progress/models.rs @@ -47,6 +47,7 @@ use sqlbuild_core::progress::main::progress_terminal_duration::progress_terminal use sqlbuild_core::progress::models::ProgressEvent; use sqlbuild_core::progress::models::ProgressProjection; use sqlbuild_core::progress::models::ProgressSettings; +use sqlbuild_core::progress::models::StatementSummary; use sqlbuild_core::terminal_events::models::TerminalRequest; use std::sync::Mutex; use std::sync::MutexGuard; @@ -130,6 +131,11 @@ impl NativeProgressProjector { lock.call_method0("release")?; result } + + /// The warehouse statements this invocation finished so far. + pub fn statement_summary(&self) -> PyResult { + Ok(self.locked()?.statement_summary().clone()) + } } #[pymethods] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index 2a86603b89..a4c1050e95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -9,4 +9,5 @@ pub(crate) mod integration_records; pub(crate) mod projection; pub(crate) mod registration; pub(crate) mod resource_ids; +pub(crate) mod statements; pub(crate) mod terminal_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs new file mode 100644 index 0000000000..dca4678ddd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs @@ -0,0 +1,136 @@ +//! The warehouse statements of a `sqb test` or `sqb scenario test` run, with their compile time. + +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTypeMethods}; +use serde_json::Value; +use sqlbuild_core::cost_allocation::main::statement_timings_sql::statement_timings_sql; +use sqlbuild_core::progress::main::format_statement_summary::format_statement_summary; +use sqlbuild_core::progress::main::statement_summary_json::statement_summary_json; +use sqlbuild_core::progress::models::{StatementSummary, StatementTiming, StatementTimings}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +use crate::bindings::_helpers::cost_telemetry::values::cost_aware; + +/// The JSON document section the report adds. +const STATEMENTS_SECTION: &str = "statements"; +/// Query identifiers per query-history read. +const QUERY_ID_BATCH: usize = 1_000; +/// Extra query-history window before the run started, for statements issued while planning. +const LOOKBACK_MARGIN_SECONDS: u64 = 600; + +/// The adapter and connection a run's statements went to, and when the run started. +pub(crate) struct StatementSource<'a, 'py> { + pub(crate) adapter_name: &'a str, + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) connection_config: &'a Bound<'py, PyAny>, + pub(crate) started: Instant, +} + +/// The statements this invocation finished, and what the warehouse reports about their time. +pub(crate) struct StatementReport { + summary: StatementSummary, + timing: StatementTiming, +} + +impl StatementReport { + /// The report of the installed projector, or nothing when no projector is installed. + pub(crate) fn read(py: Python<'_>, source: &StatementSource<'_, '_>) -> PyResult> { + let Some(projector) = current_projector(py)? else { + return Ok(None); + }; + let summary: StatementSummary = projector.get().statement_summary()?; + let timing: StatementTiming = statement_timing(py, source, &summary.query_ids)?; + Ok(Some(Self { summary, timing })) + } + + /// The summary line, without a trailing newline; nothing when no statements ran. + pub(crate) fn line(&self) -> Option { + format_statement_summary(&self.summary, &self.timing) + } + + /// `payload` with the `statements` section added. + pub(crate) fn with_section(&self, payload: &str) -> PyResult { + let invalid = |error: serde_json::Error| { + pyo3::exceptions::PyValueError::new_err(error.to_string()) + }; + let mut document: Value = serde_json::from_str(payload).map_err(invalid)?; + if let Value::Object(fields) = &mut document { + fields.insert( + STATEMENTS_SECTION.to_owned(), + statement_summary_json(&self.summary, &self.timing), + ); + } + Ok(format!( + "{}\n", + serde_json::to_string_pretty(&document).map_err(invalid)? + )) + } +} + +/// Compile and execution time from Snowflake query history; other adapters report none. +fn statement_timing( + py: Python<'_>, + source: &StatementSource<'_, '_>, + query_ids: &[String], +) -> PyResult { + let not_reported = || StatementTiming::NotReported(source.adapter_name.to_owned()); + if !cost_aware(source.adapter)? { + return Ok(not_reported()); + } + let capability: Bound<'_, PyAny> = source.adapter.call_method0("cost_capability")?; + let query_history: Bound<'_, PyAny> = + host_entry(py, ExecutorHost::CostCapability)?.getattr("SNOWFLAKE_QUERY_HISTORY")?; + if !capability.eq(query_history)? { + return Ok(not_reported()); + } + if query_ids.is_empty() { + return Ok(StatementTiming::Reported(StatementTimings::default())); + } + Ok(match read_timings(source, query_ids) { + Ok(timings) => StatementTiming::Reported(timings), + Err(error) => StatementTiming::Failed(format!( + "query history read failed: {}", + error.get_type(py).name()? + )), + }) +} + +fn read_timings( + source: &StatementSource<'_, '_>, + query_ids: &[String], +) -> PyResult { + let database: Option = match source.connection_config.downcast::() { + Ok(config) => match config.get_item("database")? { + Some(database) if !database.is_none() => Some(database.str()?.extract()?), + _ => None, + }, + Err(_) => None, + }; + let lookback_seconds: u64 = source.started.elapsed().as_secs() + LOOKBACK_MARGIN_SECONDS; + let connection: Bound<'_, PyAny> = source + .adapter + .call_method1("connect", (source.connection_config,))?; + let read = || -> PyResult { + let mut timings: StatementTimings = StatementTimings::default(); + for batch in query_ids.chunks(QUERY_ID_BATCH) { + let sql: String = statement_timings_sql(database.as_deref(), batch, lookback_seconds); + let (count, compile_ms, execution_ms): (u64, f64, f64) = connection + .call_method1("execute", (sql,))? + .call_method0("fetchone")? + .extract()?; + timings.timed_statements += count; + timings.compile_ms += compile_ms; + timings.execution_ms += execution_ms; + } + Ok(timings) + }; + let timings: PyResult = read(); + let closed: PyResult> = source.adapter.call_method1("close", (&connection,)); + let timings: StatementTimings = timings?; + closed?; + Ok(timings) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 1422bd6b50..f57a68e517 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -26,6 +26,7 @@ use crate::bindings::_helpers::change_selection::selection::{ }; use crate::bindings::_helpers::result_output::documents::format_scenario_execution_json_with; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; +use crate::bindings::_helpers::result_output::statements::{StatementReport, StatementSource}; use crate::bindings::_helpers::scenario_command::conversions::scenario_result_row; use crate::bindings::_helpers::scenario_command::presentation::{ CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, @@ -168,6 +169,18 @@ fn finish( if !failures.is_empty() { write(&run.stream, &format!("{failures}\n"))?; } + let report: Option = StatementReport::read( + py, + &StatementSource { + adapter_name: &run.adapter_name, + adapter: &run.adapter, + connection_config: &run.connection_config, + started, + }, + )?; + if let Some(line) = report.as_ref().and_then(StatementReport::line) { + write(&run.stream, &format!("{line}\n"))?; + } flush(&run.stream)?; write_completion(&run.stream, (&run.project.namespace, succeeded))?; let namespace: &ScenarioNamespace = &run.project.namespace; @@ -194,7 +207,11 @@ fn finish( )?), )?, run.since_json.as_deref(), - )?, + ) + .and_then(|payload| match &report { + Some(report) => report.with_section(&payload), + None => Ok(payload), + })?, ) .into_any(); write_execution_json_output( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index a721a2c9df..d78e818b4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; use std::sync::Arc; +use std::time::Instant; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -45,6 +46,7 @@ use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; +use crate::bindings::_helpers::result_output::statements::{StatementReport, StatementSource}; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; use crate::bindings::_helpers::sql_test_command::python_tests::{ @@ -656,6 +658,7 @@ pub(crate) fn timeout_record<'py>( fn complete( invocation: &Bound<'_, PyAny>, (results, since_json): (&[SqlTestExecutionResult>], Option<&str>), + started: Instant, ) -> PyResult<(i32, String)> { let py: Python<'_> = invocation.py(); let mut failures: Vec = Vec::new(); @@ -683,10 +686,26 @@ fn complete( style, ),), )?; + let report: Option = StatementReport::read( + py, + &StatementSource { + adapter_name: &invocation.getattr("adapter_name")?.extract::()?, + adapter: &invocation.getattr("adapter")?, + connection_config: &invocation.getattr("connection_config")?, + started, + }, + )?; + if let Some(line) = report.as_ref().and_then(StatementReport::line) { + stream.call_method1("write", (format!("\n{line}\n"),))?; + } stream.call_method0("flush")?; + let payload: String = with_since_section(test_execution_json(py, results)?, since_json)?; Ok(( test_exit_code(passed, results.len()), - with_since_section(test_execution_json(py, results)?, since_json)?, + match report { + Some(report) => report.with_section(&payload)?, + None => payload, + }, )) } @@ -800,6 +819,7 @@ fn run_sql_tests<'py>( (modules, resolve): (Option>, bool), ) -> PyResult<(i32, Vec, Option)> { let py: Python<'py> = request.py(); + let started: Instant = Instant::now(); let inspect_only: bool = native_field(request, "inspect")?.is_truthy()?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; let project_dir: Bound<'_, PyAny> = native_field(invocation, "effective_project_dir")?; @@ -894,7 +914,7 @@ fn run_sql_tests<'py>( &pipeline.getattr("plan_output")?, &results, )?; - let (code, payload): (i32, String) = complete(invocation, (&results, since_json.as_deref()))?; + let (code, payload): (i32, String) = complete(invocation, (&results, since_json.as_deref()), started)?; Ok((code, python_tests, Some(payload))) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 9d449d32ef..c3f598dc52 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -125,10 +125,12 @@ daily_revenue_minimal PASS 0.84s expect assertion no_negative_revenue PASS PASS=1 FAIL=0 TOTAL=1 (0.91s) +Statements: 9 (CREATE 3, SELECT 3, DROP 2, ALTER 1); compile time not reported by duckdb ``` Each scenario row shows its wall time and the summary shows the total. `--json` reports the same -values as `scenarios[].duration_ms` and `summary.duration_ms`. +values as `scenarios[].duration_ms` and `summary.duration_ms`. The `Statements:` line and the JSON +`statements` section are described in [Statement summary](test.md#statement-summary). Local scenarios add ERROR and SKIP statuses: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 59804b6c8d..13dce768af 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/cli/test/ - Usage - Flags +- Statement summary - Python tests - Run only affected tests - Examples @@ -64,6 +65,30 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Statement summary + +After the pass/fail summary, `sqb test` and `sqb scenario test` print how many warehouse statements +the run sent, by leading keyword, and how long the warehouse spent compiling and executing them: + +``` +Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s +``` + +With `--json`, the document gains a `statements` section with `total`, `by_kind` and +`compile_time`. + +| Adapter | Compile and execution time | +| --- | --- | +| Snowflake | Read from `INFORMATION_SCHEMA.QUERY_HISTORY` after the run; `compile_time` has `compile_ms`, `execution_ms`, `timed_statements` and `identified_statements` | +| Other adapters | Not reported; the line says `compile time not reported by ` and `compile_time` is `null` | + +Details: + +- On Snowflake, statements that query history does not return yet are left out of the times, and + the line says how many were found, such as `(10 of 12 statements in query history)`. +- When the query-history read fails, the counts still print and the line says + `compile time unavailable (...)`; the run's exit code does not change. + ## Python tests Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 9f8e2bd4af..7b37ec6cb3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -638,3 +638,14 @@ class PythonTestsJsonE2ETestCase: expected_status: str expected_python_status: str expected_python_files: tuple[str, ...] + + +@dataclass(frozen=True) +class StatementSummaryE2ETestCase: + """A `sqb test` run whose statement summary must count each finished statement once.""" + + description: str + expected_doubled: tuple[int, ...] + expected_exit_code: int + expected_by_kind: Mapping[str, int] + expected_line: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py new file mode 100644 index 0000000000..f33139643d --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py @@ -0,0 +1,126 @@ +"""`sqb test` and `sqb scenario test` report the warehouse statements they ran, by kind.""" + +import json +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( + build_scenario_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + StatementSummaryE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: amount\n type: INTEGER\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n\n" + 'SELECT order_id, amount * 2 AS doubled FROM __source("raw_orders")\n' + ), +} + + +def _test_file(name: str, doubled: int) -> str: + return ( + f'TEST (name "{name}");\n\nWITH\n' + "__source__raw_orders AS (\n SELECT 1 AS order_id, 5 AS amount\n),\n" + f"__expected__orders AS (\n SELECT 1 AS order_id, {doubled} AS doubled\n)\nSELECT 1\n" + ) + + +@pytest.mark.parametrize( + "test_case", + ( + # Each test of one real model stores the model's output (CREATE), compares it with the + # expected rows (WITH) and drops it (DROP). + StatementSummaryE2ETestCase( + description="two passing tests count three statements each", + expected_doubled=(10, 10), + expected_exit_code=0, + expected_by_kind={"CREATE": 2, "DROP": 2, "WITH": 2}, + expected_line=( + "Statements: 6 (CREATE 2, DROP 2, WITH 2); compile time not reported by duckdb" + ), + ), + # A failed comparison also reads its unexpected and missing sample rows (two more WITH). + StatementSummaryE2ETestCase( + description="a failing test's statements count too", + expected_doubled=(10, 99), + expected_exit_code=1, + expected_by_kind={"CREATE": 2, "DROP": 2, "WITH": 4}, + expected_line=( + "Statements: 8 (WITH 4, CREATE 2, DROP 2); compile time not reported by duckdb" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_sql_tests_when_running_then_statement_summary_counts_each_statement( + tmp_path: Path, test_case: StatementSummaryE2ETestCase +) -> None: + files: dict[str, str] = dict(_PROJECT_FILES) + for index, doubled in enumerate(test_case.expected_doubled): + files[f"tests/unit/test_orders_{index}.sql"] = _test_file(f"orders_{index}", doubled) + project: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="statement_summary", repo_files=files + ) + + text_run: CompletedProcess[str] = run_sqb(command=("--no-color", "test"), project_dir=project) + json_run: CompletedProcess[str] = run_sqb( + command=("--no-color", "test", "--json"), project_dir=project + ) + + assert text_run.returncode == test_case.expected_exit_code, text_run.stdout + text_run.stderr + assert test_case.expected_line in text_run.stdout.splitlines(), text_run.stdout + assert json_run.returncode == test_case.expected_exit_code, json_run.stderr + assert json.loads(json_run.stdout)["statements"] == { + "total": sum(test_case.expected_by_kind.values()), + "by_kind": dict(test_case.expected_by_kind), + "compile_time": None, + } + assert test_case.expected_line in json_run.stderr.splitlines(), json_run.stderr + + +def test_given_scenarios_when_running_with_json_then_statement_summary_matches_progress_line( + tmp_path: Path, +) -> None: + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="scenario_statements", + repo_files=build_scenario_project_files(), + ) + + result: CompletedProcess[str] = run_sqb( + command=("--no-color", "scenario", "test", "--json"), project_dir=project + ) + + assert result.returncode == 1, result.stderr + statements: dict[str, object] = json.loads(result.stdout)["statements"] + by_kind: dict[str, int] = statements["by_kind"] # type: ignore[assignment] + assert statements["total"] == sum(by_kind.values()) and {"CREATE", "DROP"} <= set(by_kind) + assert statements["compile_time"] is None + kinds: str = ", ".join( + f"{kind} {count}" + for kind, count in sorted(by_kind.items(), key=lambda item: (-item[1], item[0])) + ) + assert ( + f"Statements: {statements['total']} ({kinds}); compile time not reported by duckdb" + in result.stderr.splitlines() + ), result.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index 43e2c46638..922bc1c2d4 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -118,10 +118,12 @@ daily_revenue_minimal PASS 0.84s expect assertion no_negative_revenue PASS PASS=1 FAIL=0 TOTAL=1 (0.91s) +Statements: 9 (CREATE 3, SELECT 3, DROP 2, ALTER 1); compile time not reported by duckdb ``` Each scenario row shows its wall time and the summary shows the total. `--json` reports the same -values as `scenarios[].duration_ms` and `summary.duration_ms`. +values as `scenarios[].duration_ms` and `summary.duration_ms`. The `Statements:` line and the JSON +`statements` section are described in [Statement summary](/docs/cli/test#statement-summary). Local scenarios add ERROR and SKIP statuses: diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index 67cfd4fd04..8da5b0aa45 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -55,6 +55,30 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Statement summary + +After the pass/fail summary, `sqb test` and `sqb scenario test` print how many warehouse statements +the run sent, by leading keyword, and how long the warehouse spent compiling and executing them: + +``` +Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s +``` + +With `--json`, the document gains a `statements` section with `total`, `by_kind` and +`compile_time`. + +| Adapter | Compile and execution time | +| --- | --- | +| Snowflake | Read from `INFORMATION_SCHEMA.QUERY_HISTORY` after the run; `compile_time` has `compile_ms`, `execution_ms`, `timed_statements` and `identified_statements` | +| Other adapters | Not reported; the line says `compile time not reported by ` and `compile_time` is `null` | + +Details: + +- On Snowflake, statements that query history does not return yet are left out of the times, and + the line says how many were found, such as `(10 of 12 statements in query history)`. +- When the query-history read fails, the counts still print and the line says + `compile time unavailable (...)`; the run's exit code does not change. + ## Python tests Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each From 0adc634d40ff05ee404f0b2e9580c24ab9237555 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:43:47 +0000 Subject: [PATCH 10/15] fix: keep commands working with a CTX-templated target schema; bound the statement history read --- .../_helpers/compile_driver/targets.rs | 11 +++- .../_helpers/result_output/statements.rs | 31 ++++++---- .../sqlbuild/references/docs/cli/test.md | 3 +- .../docs/concepts/python-nodes/overview.md | 2 +- .../cli/commands/main/build/test_build.py | 60 +++++++++++++++++++ website/src/content/docs/docs/cli/test.mdx | 3 +- .../docs/concepts/python-nodes/overview.mdx | 2 +- 7 files changed, 95 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 648921f4b1..d46aa32b88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet}; +use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -473,11 +474,17 @@ pub(crate) fn target_settings<'py>( )? }; let none: Bound<'py, PyAny> = py.None().into_bound(py); - let (_, loader_schema): Namespace = managed_source_namespace( + // A loader location that only resolves inside a model, such as a `${CTX:...}` target schema, + // has no value here; managed loads still resolve and report it themselves. + let loader_schema: Option = match managed_source_namespace( (none.clone(), none), (&target_config, connection, effective_vars), adapter, - )?; + ) { + Ok((_, schema)) => schema, + Err(error) if error.is_instance_of::(py) => None, + Err(error) => return Err(error), + }; Ok(TargetSettingsValues { name: target_name.extract()?, database, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs index dca4678ddd..7bce151b5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs @@ -2,6 +2,7 @@ use std::time::Instant; +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTypeMethods}; use serde_json::Value; @@ -21,6 +22,9 @@ const STATEMENTS_SECTION: &str = "statements"; const QUERY_ID_BATCH: usize = 1_000; /// Extra query-history window before the run started, for statements issued while planning. const LOOKBACK_MARGIN_SECONDS: u64 = 600; +/// Connection and statement limits of the history read, so it cannot stall a finished run. +const CONNECTION_LIMITS: [(&str, u64); 2] = [("login_timeout", 5), ("network_timeout", 15)]; +const STATEMENT_TIMEOUT_SECONDS: &str = "10"; /// The adapter and connection a run's statements went to, and when the run started. pub(crate) struct StatementSource<'a, 'py> { @@ -92,10 +96,11 @@ fn statement_timing( } Ok(match read_timings(source, query_ids) { Ok(timings) => StatementTiming::Reported(timings), - Err(error) => StatementTiming::Failed(format!( + Err(error) if error.is_instance_of::(py) => StatementTiming::Failed(format!( "query history read failed: {}", error.get_type(py).name()? )), + Err(error) => return Err(error), }) } @@ -103,23 +108,27 @@ fn read_timings( source: &StatementSource<'_, '_>, query_ids: &[String], ) -> PyResult { - let database: Option = match source.connection_config.downcast::() { - Ok(config) => match config.get_item("database")? { - Some(database) if !database.is_none() => Some(database.str()?.extract()?), - _ => None, - }, - Err(_) => None, + let py: Python<'_> = source.adapter.py(); + let config: Bound<'_, PyDict> = source.connection_config.downcast::()?.copy()?; + let database: Option = match config.get_item("database")? { + Some(database) if !database.is_none() => Some(database.str()?.extract()?), + _ => None, }; + for (name, limit) in CONNECTION_LIMITS { + config.set_item(name, limit)?; + } + let parameters: Bound<'_, PyDict> = PyDict::new(py); + parameters.set_item("STATEMENT_TIMEOUT_IN_SECONDS", STATEMENT_TIMEOUT_SECONDS)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("statement_params", parameters)?; let lookback_seconds: u64 = source.started.elapsed().as_secs() + LOOKBACK_MARGIN_SECONDS; - let connection: Bound<'_, PyAny> = source - .adapter - .call_method1("connect", (source.connection_config,))?; + let connection: Bound<'_, PyAny> = source.adapter.call_method1("connect", (config,))?; let read = || -> PyResult { let mut timings: StatementTimings = StatementTimings::default(); for batch in query_ids.chunks(QUERY_ID_BATCH) { let sql: String = statement_timings_sql(database.as_deref(), batch, lookback_seconds); let (count, compile_ms, execution_ms): (u64, f64, f64) = connection - .call_method1("execute", (sql,))? + .call_method("execute", (sql,), Some(&options))? .call_method0("fetchone")? .extract()?; timings.timed_statements += count; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 13dce768af..8ee38ea28f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -68,7 +68,7 @@ many of them timed out. A timeout still fails the run: `sqb test` exits non-zero ## Statement summary After the pass/fail summary, `sqb test` and `sqb scenario test` print how many warehouse statements -the run sent, by leading keyword, and how long the warehouse spent compiling and executing them: +the SQL tests or scenarios sent, by leading keyword, and how long the warehouse spent compiling and executing them: ``` Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s @@ -88,6 +88,7 @@ Details: the line says how many were found, such as `(10 of 12 statements in query history)`. - When the query-history read fails, the counts still print and the line says `compile time unavailable (...)`; the run's exit code does not change. +- Statements from [Python tests](#python-tests) are not counted. ## Python tests diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 08b2cd3210..bd1768654e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -138,7 +138,7 @@ Read the active target's locations from `ctx.target_config` instead of keeping e | `name` | Active target name, or `None` without targets | | `database` | Database models build into, or `None` when the adapter has none or the value is `preserve` | | `schema` | Schema models build into, or `None` when it is `preserve` | -| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema` | +| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema`; `None` when that schema uses a `${CTX:...}` template | | `database_preserved` / `schema_preserved` | `True` when the target sets that value to `preserve` | ```python diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index 0e11ed812d..cdeee5aeec 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -1929,6 +1929,66 @@ def test_given_local_target_override_when_running_python_then_every_context_sees assert all(line == test_case.expected_settings for line in recorded), recorded +def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_loader_schema( + tmp_path: Path, +) -> None: + # A per-model schema has no single loader location outside a model; it must not fail commands. + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="ctx_target_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "ctx_target_project"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "ctx_target_project.duckdb"\n\n' + "[targets.dev]\n" + 'schema = "s_${CTX:model.name}"\n' + ), + "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", + "python/recorder.py": _TARGET_CONFIG_RECORDER, + "python/loaders/raw.py": ( + "from python.recorder import record\n" + "from sqlbuild.loaders import loader\n\n" + "@loader\n" + "def raw_orders(ctx):\n" + ' """Test loader raw_orders."""\n' + " record(ctx, 'loader')\n" + " return [{'order_id': 1}]\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Test source raw_orders.\n" + " managed: true\n" + " schema: raw\n" + " write_strategy: table\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + ), + }, + ) + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=("--no-color", *command), project_dir=project_dir) + for command in (("build", "--select", "orders"), ("load",), ("check",)) + ] + + for result in results: + assert result.returncode == 0, result.stdout + result.stderr + db_path: Path = project_dir / "ctx_target_project.duckdb" + assert table_exists(db_path=db_path, table_name="orders", schema="s_orders") + assert table_exists(db_path=db_path, table_name="raw_orders", schema="raw") + recorded: list[dict[str, object]] = [ + json.loads(line) + for line in (project_dir / "target_config.jsonl").read_text(encoding="utf-8").splitlines() + ] + assert [(line["node"], line["name"], line["loader_schema"]) for line in recorded] == [ + ("loader", "dev", None) + ], recorded + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index 8da5b0aa45..fb168a62e5 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -58,7 +58,7 @@ many of them timed out. A timeout still fails the run: `sqb test` exits non-zero ## Statement summary After the pass/fail summary, `sqb test` and `sqb scenario test` print how many warehouse statements -the run sent, by leading keyword, and how long the warehouse spent compiling and executing them: +the SQL tests or scenarios sent, by leading keyword, and how long the warehouse spent compiling and executing them: ``` Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s @@ -78,6 +78,7 @@ Details: the line says how many were found, such as `(10 of 12 statements in query history)`. - When the query-history read fails, the counts still print and the line says `compile time unavailable (...)`; the run's exit code does not change. +- Statements from [Python tests](#python-tests) are not counted. ## Python tests diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 2ef12f10d4..87c93836f3 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -122,7 +122,7 @@ Read the active target's locations from `ctx.target_config` instead of keeping e | `name` | Active target name, or `None` without targets | | `database` | Database models build into, or `None` when the adapter has none or the value is `preserve` | | `schema` | Schema models build into, or `None` when it is `preserve` | -| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema` | +| `loader_schema` | Schema a managed source without its own `schema` loads into: `loader_schema`, else the target's `schema`; `None` when that schema uses a `${CTX:...}` template | | `database_preserved` / `schema_preserved` | `True` when the target sets that value to `preserve` | ```python From 918bba679c75426d63eb23b091501c78d571b252 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:01:57 +0000 Subject: [PATCH 11/15] feat: pass the active target to provider setup in build, check and load --- .../main/statement_timings_sql.rs | 3 +- .../tests/test_statement_timings.rs | 47 ++- .../src/cost_allocation/tests/test_types.rs | 7 + .../src/progress/_helpers/statements.rs | 14 +- .../progress/main/consume_progress_event.rs | 3 +- .../src/progress/tests/helpers.rs | 23 ++ .../src/progress/tests/test_progress.rs | 321 +++++++++++------- .../src/progress/tests/test_types.rs | 17 +- .../_helpers/contract_records/constants.rs | 6 + .../records/_helpers/native_classes/ids.rs | 1 + .../sqlbuild-python-base/src/records/types.rs | 1 + .../_helpers/build_command/command.rs | 31 +- .../bindings/_helpers/check/check_command.rs | 11 +- .../_helpers/compile_driver/targets.rs | 41 ++- .../bindings/_helpers/load/load_command.rs | 46 +-- .../src/bindings/_helpers/native_host/mod.rs | 1 + .../statement_report.rs} | 21 +- .../bindings/_helpers/result_output/mod.rs | 1 - .../_helpers/scenario_command/test_command.rs | 2 +- .../_helpers/sql_test_command/test_command.rs | 27 +- .../docs/concepts/python-nodes/providers.md | 17 + src/sqlbuild/_native.pyi | 6 + .../cli/commands/main/build/_test_types.py | 10 + .../cli/commands/main/build/test_build.py | 52 ++- .../cli/commands/main/test/_test_types.py | 9 + .../cli/commands/main/test/helpers.py | 10 + .../main/test/test_statement_summary.py | 32 +- .../docs/concepts/python-nodes/providers.mdx | 17 + 28 files changed, 548 insertions(+), 229 deletions(-) rename crates/sqlbuild-python/src/bindings/_helpers/{result_output/statements.rs => native_host/statement_report.rs} (90%) diff --git a/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs index af2525c090..6ccc8ac2a0 100644 --- a/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs +++ b/crates/sqlbuild-core/src/cost_allocation/main/statement_timings_sql.rs @@ -3,8 +3,7 @@ use crate::cost_allocation::_helpers::history_function::query_history_function; use crate::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; -/// One query-history read summing compile and execution milliseconds of `query_ids`, among -/// queries that ended in the last `lookback_seconds`. +/// One query-history read summing compile and execution milliseconds of `query_ids` that ended in the last `lookback_seconds`. #[must_use] pub fn statement_timings_sql( database: Option<&str>, diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs index ef03ff93a7..f6993b2d8b 100644 --- a/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_statement_timings.rs @@ -1,14 +1,14 @@ use crate::cost_allocation::main::statement_timings_sql::statement_timings_sql; +use crate::cost_allocation::tests::test_types::StatementTimingsSqlTestCase; #[test] fn given_query_ids_when_rendering_timings_read_then_ids_are_quoted_and_database_qualified() { - let query_ids: Vec = vec!["01a-1".to_owned(), "it's".to_owned()]; - - let sql: String = statement_timings_sql(Some("analytics\"db"), &query_ids, 720); - - assert_eq!( - sql, - "SELECT + let test_cases = [ + StatementTimingsSqlTestCase { + description: "a quote in an id is doubled and the database is quoted upper case", + database: Some("analytics\"db"), + query_ids: vec!["01a-1", "it's"], + expected_sql: "SELECT COUNT(*), COALESCE(SUM(COMPILATION_TIME), 0), COALESCE(SUM(EXECUTION_TIME), 0) @@ -16,6 +16,35 @@ FROM TABLE(\"ANALYTICS\"\"DB\".INFORMATION_SCHEMA.QUERY_HISTORY( END_TIME_RANGE_START => DATEADD('second', -720, CURRENT_TIMESTAMP()), RESULT_LIMIT => 10000 )) -WHERE QUERY_ID IN ('01a-1', 'it''s')" - ); +WHERE QUERY_ID IN ('01a-1', 'it''s')", + }, + StatementTimingsSqlTestCase { + description: "without a database the session's information schema is read", + database: None, + query_ids: vec!["q1"], + expected_sql: "SELECT + COUNT(*), + COALESCE(SUM(COMPILATION_TIME), 0), + COALESCE(SUM(EXECUTION_TIME), 0) +FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY( + END_TIME_RANGE_START => DATEADD('second', -720, CURRENT_TIMESTAMP()), + RESULT_LIMIT => 10000 +)) +WHERE QUERY_ID IN ('q1')", + }, + ]; + + for test_case in test_cases { + let query_ids: Vec = test_case + .query_ids + .iter() + .map(|query_id| (*query_id).to_owned()) + .collect(); + assert_eq!( + statement_timings_sql(test_case.database, &query_ids, 720), + test_case.expected_sql, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs index cefbc507fe..9c6ffe2fee 100644 --- a/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs @@ -12,3 +12,10 @@ pub(crate) struct AllocationTestCase { pub(crate) expected_seconds: &'static str, pub(crate) expected_usd: &'static str, } + +pub(crate) struct StatementTimingsSqlTestCase { + pub(crate) description: &'static str, + pub(crate) database: Option<&'static str>, + pub(crate) query_ids: Vec<&'static str>, + pub(crate) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-core/src/progress/_helpers/statements.rs b/crates/sqlbuild-core/src/progress/_helpers/statements.rs index f536c2cdce..296874db69 100644 --- a/crates/sqlbuild-core/src/progress/_helpers/statements.rs +++ b/crates/sqlbuild-core/src/progress/_helpers/statements.rs @@ -144,12 +144,16 @@ pub(crate) fn consume_statement_terminal( } /// Count a finished statement by kind, keeping its query identifier. -pub(crate) fn count_statement(projection: &mut ProgressProjection, event: &ProgressEvent) { - let query_id: Option = statement_query_id(projection, event); - let statements = &mut projection.statements; - *statements +pub(crate) fn count_statement( + mut projection: ProgressProjection, + event: &ProgressEvent, +) -> ProgressProjection { + let query_id: Option = statement_query_id(&projection, event); + *projection + .statements .by_kind .entry(event.payload.statement_kind.clone()) .or_default() += 1; - statements.query_ids.extend(query_id); + projection.statements.query_ids.extend(query_id); + projection } diff --git a/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs index b2aaa9c108..744b1fa2bb 100644 --- a/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs +++ b/crates/sqlbuild-core/src/progress/main/consume_progress_event.rs @@ -46,8 +46,7 @@ pub fn consume_progress_event( let line: Option = statement_heartbeat_line(&projection, event); (projection, line) } else if event_type == STATEMENT_COMPLETED || event_type == STATEMENT_FAILED { - count_statement(&mut projection, event); - consume_statement_terminal(projection, event) + consume_statement_terminal(count_statement(projection, event), event) } else { (projection, None) }; diff --git a/crates/sqlbuild-core/src/progress/tests/helpers.rs b/crates/sqlbuild-core/src/progress/tests/helpers.rs index bdbd5d6d28..a3367b93a7 100644 --- a/crates/sqlbuild-core/src/progress/tests/helpers.rs +++ b/crates/sqlbuild-core/src/progress/tests/helpers.rs @@ -115,3 +115,26 @@ pub(super) fn audit_rows(test_case: &AuditRowTestCase) -> Vec { } lines } + +/// A statement event of `kind`, inside the `orders` attempt `attempt_id` or outside any resource. +pub(super) fn statement_event( + (event_id, event_type): (&str, &str), + (statement_id, kind, query_id): (&str, &str, Option<&str>), + attempt_id: Option<&str>, +) -> ProgressEvent { + ProgressEvent { + event_id: event_id.to_owned(), + event_type: event_type.to_owned(), + resource_id: attempt_id.map(|_| "model:orders".to_owned()), + resource_attempt_id: attempt_id.map(str::to_owned), + statement_id: Some(statement_id.to_owned()), + payload: ProgressPayload { + resource_name: attempt_id.map(|_| "orders".to_owned()), + resource_kind: attempt_id.map(|_| "table".to_owned()), + statement_kind: kind.to_owned(), + query_id: query_id.map(str::to_owned), + ..ProgressPayload::default() + }, + ..ProgressEvent::default() + } +} diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress.rs b/crates/sqlbuild-core/src/progress/tests/test_progress.rs index bfcca92e47..0ea684b0e0 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress.rs @@ -8,9 +8,11 @@ use crate::progress::models::{ StatementTiming, StatementTimings, }; use crate::progress::tests::helpers::{ - attempt_event, audit_rows, operation_event, orders_claim, plain, project_case, + attempt_event, audit_rows, operation_event, orders_claim, plain, project_case, statement_event, +}; +use crate::progress::tests::test_types::{ + AuditRowTestCase, ProjectionTestCase, StatementCountTestCase, StatementSummaryTextTestCase, }; -use crate::progress::tests::test_types::{AuditRowTestCase, ProjectionTestCase}; #[test] fn given_lifecycle_facts_when_projecting_then_progress_lines_match() { @@ -274,64 +276,111 @@ fn given_standalone_audit_when_projecting_then_row_names_target_and_outcome() { } } -/// A statement event of `kind`, inside the `orders` attempt when `known` and outside any resource otherwise. -fn statement_event( - (event_id, event_type): (&str, &str), - (statement_id, kind, query_id): (&str, &str, Option<&str>), - known: bool, -) -> ProgressEvent { - let base: ProgressEvent = if known { - attempt_event(event_id, event_type, ("a1", None)) - } else { - ProgressEvent { - event_id: event_id.to_owned(), - event_type: event_type.to_owned(), - ..ProgressEvent::default() - } - }; - ProgressEvent { - statement_id: Some(statement_id.to_owned()), - payload: ProgressPayload { - statement_kind: kind.to_owned(), - query_id: query_id.map(str::to_owned), - ..base.payload - }, - ..base - } -} - #[test] fn given_statement_terminals_when_projecting_then_every_finished_statement_is_counted_once() { - let events: Vec = vec![ - attempt_event("r1", "resource_attempt_started", ("a1", None)), - statement_event(("s1", "statement_started"), ("st1", "CREATE", None), true), - statement_event(("s2", "statement_submitted"), ("st1", "CREATE", Some("q1")), true), - // The terminal omits the query identifier its submission reported. - statement_event(("s3", "statement_completed"), ("st1", "CREATE", None), true), - // A redelivered terminal is not counted again. - statement_event(("s3", "statement_completed"), ("st1", "CREATE", None), true), - // Statements outside any known resource, with and without a start, still count. - statement_event(("s4", "statement_completed"), ("st2", "WITH", Some("q2")), false), - statement_event(("s5", "statement_failed"), ("st3", "DROP", None), false), - statement_event(("s6", "statement_completed"), ("st4", "WITH", Some("q3")), false), - // Starts, submissions and heartbeats alone are not finished statements. - statement_event(("s7", "statement_started"), ("st5", "SELECT", None), false), - statement_event(("s8", "statement_heartbeat"), ("st5", "SELECT", Some("q5")), false), + let test_cases = [ + StatementCountTestCase { + description: "a known resource's terminal takes the query id its submission reported", + events: vec![ + attempt_event("r1", "resource_attempt_started", ("a1", None)), + statement_event( + ("s1", "statement_started"), + ("st1", "CREATE", None), + Some("a1"), + ), + statement_event( + ("s2", "statement_submitted"), + ("st1", "CREATE", Some("q1")), + Some("a1"), + ), + statement_event( + ("s3", "statement_completed"), + ("st1", "CREATE", None), + Some("a1"), + ), + ], + expected_by_kind: vec![("CREATE", 1)], + expected_query_ids: vec!["q1"], + }, + StatementCountTestCase { + description: "a redelivered terminal is counted once", + events: vec![ + statement_event( + ("s1", "statement_completed"), + ("st1", "WITH", Some("q1")), + None, + ), + statement_event( + ("s1", "statement_completed"), + ("st1", "WITH", Some("q1")), + None, + ), + ], + expected_by_kind: vec![("WITH", 1)], + expected_query_ids: vec!["q1"], + }, + StatementCountTestCase { + description: "statements outside known resources count, failed ones too", + events: vec![ + statement_event( + ("s1", "statement_completed"), + ("st1", "WITH", Some("q1")), + None, + ), + statement_event(("s2", "statement_failed"), ("st2", "DROP", None), None), + statement_event( + ("s3", "statement_completed"), + ("st3", "WITH", Some("q3")), + None, + ), + ], + expected_by_kind: vec![("DROP", 1), ("WITH", 2)], + expected_query_ids: vec!["q1", "q3"], + }, + StatementCountTestCase { + description: "starts, submissions and heartbeats alone are not finished statements", + events: vec![ + statement_event(("s1", "statement_started"), ("st1", "SELECT", None), None), + statement_event( + ("s2", "statement_submitted"), + ("st1", "SELECT", Some("q1")), + None, + ), + statement_event( + ("s3", "statement_heartbeat"), + ("st1", "SELECT", Some("q1")), + None, + ), + ], + expected_by_kind: vec![], + expected_query_ids: vec![], + }, ]; - let mut projection: ProgressProjection = ProgressProjection::new(plain()); - for event in &events { - projection = consume_progress_event(projection, event).0; + + for test_case in test_cases { + let mut projection: ProgressProjection = ProgressProjection::new(plain()); + for event in &test_case.events { + projection = consume_progress_event(projection, event).0; + } + let summary: &StatementSummary = projection.statement_summary(); + assert_eq!( + (summary.by_kind.clone(), summary.query_ids.clone()), + ( + test_case + .expected_by_kind + .iter() + .map(|(kind, count)| ((*kind).to_owned(), *count)) + .collect::>(), + test_case + .expected_query_ids + .iter() + .map(|query_id| (*query_id).to_owned()) + .collect::>(), + ), + "{}", + test_case.description + ); } - let summary: &StatementSummary = projection.statement_summary(); - assert_eq!( - summary.by_kind, - BTreeMap::from([ - ("CREATE".to_owned(), 1), - ("DROP".to_owned(), 1), - ("WITH".to_owned(), 2), - ]) - ); - assert_eq!(summary.query_ids, ["q1", "q2", "q3"]); } #[test] @@ -345,80 +394,112 @@ fn given_statement_summary_when_formatting_then_line_and_json_report_kinds_and_t ]), query_ids: (0..12).map(|index| format!("q{index}")).collect(), }; - let complete: StatementTiming = StatementTiming::Reported(StatementTimings { - compile_ms: 1540.0, - execution_ms: 260.0, - timed_statements: 12, - }); - let partial: StatementTiming = StatementTiming::Reported(StatementTimings { - compile_ms: 1000.0, - execution_ms: 0.0, - timed_statements: 10, - }); - let cases: [(&str, &StatementSummary, StatementTiming, Option<&str>); 5] = [ - ( - "kinds run from most to least frequent, ties by name", - &summary, - complete.clone(), - Some( + let reported = |compile_ms: f64, execution_ms: f64, timed_statements: u64| { + StatementTiming::Reported(StatementTimings { + compile_ms, + execution_ms, + timed_statements, + }) + }; + let test_cases = [ + StatementSummaryTextTestCase { + description: "kinds run from most to least frequent, ties by name", + summary: summary.clone(), + timing: reported(1540.0, 260.0, 12), + expected_line: Some( "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.5s, execution 0.3s", ), - ), - ( - "statements missing from query history are called out", - &summary, - partial, - Some( + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": { + "source": "snowflake_query_history", + "compile_ms": 1540.0, + "execution_ms": 260.0, + "timed_statements": 12, + "identified_statements": 12, + }, + }), + }, + StatementSummaryTextTestCase { + description: "statements missing from query history are called out", + summary: summary.clone(), + timing: reported(1000.0, 0.0, 10), + expected_line: Some( "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile 1.0s, execution 0.0s (10 of 12 statements in query history)", ), - ), - ( - "an adapter without query history says so", - &summary, - StatementTiming::NotReported("duckdb".to_owned()), - Some( + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": { + "source": "snowflake_query_history", + "compile_ms": 1000.0, + "execution_ms": 0.0, + "timed_statements": 10, + "identified_statements": 12, + }, + }), + }, + StatementSummaryTextTestCase { + description: "an adapter without query history says so", + summary: summary.clone(), + timing: StatementTiming::NotReported("duckdb".to_owned()), + expected_line: Some( "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile time not reported by duckdb", ), - ), - ( - "a failed read says why", - &summary, - StatementTiming::Failed("query history read failed".to_owned()), - Some( + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": null, + }), + }, + StatementSummaryTextTestCase { + description: "a failed read says why", + summary: summary.clone(), + timing: StatementTiming::Failed("query history read failed".to_owned()), + expected_line: Some( "Statements: 12 (SELECT 7, CREATE 2, DROP 2, ALTER 1); compile time unavailable (query history read failed)", ), - ), - ( - "a run without statements has no line", - &StatementSummary::default(), - complete.clone(), - None, - ), + expected_json: serde_json::json!({ + "total": 12, + "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, + "compile_time": { + "source": "snowflake_query_history", + "error": "query history read failed", + }, + }), + }, + StatementSummaryTextTestCase { + description: "a run without statements has no line", + summary: StatementSummary::default(), + timing: reported(0.0, 0.0, 0), + expected_line: None, + expected_json: serde_json::json!({ + "total": 0, + "by_kind": {}, + "compile_time": { + "source": "snowflake_query_history", + "compile_ms": 0.0, + "execution_ms": 0.0, + "timed_statements": 0, + "identified_statements": 0, + }, + }), + }, ]; - for (description, summary, timing, expected) in cases { + + for test_case in test_cases { assert_eq!( - format_statement_summary(summary, &timing).as_deref(), - expected, - "{description}" + ( + format_statement_summary(&test_case.summary, &test_case.timing), + statement_summary_json(&test_case.summary, &test_case.timing), + ), + ( + test_case.expected_line.map(str::to_owned), + test_case.expected_json, + ), + "{}", + test_case.description ); } - assert_eq!( - statement_summary_json(&summary, &complete), - serde_json::json!({ - "total": 12, - "by_kind": {"ALTER": 1, "CREATE": 2, "DROP": 2, "SELECT": 7}, - "compile_time": { - "source": "snowflake_query_history", - "compile_ms": 1540.0, - "execution_ms": 260.0, - "timed_statements": 12, - "identified_statements": 12, - }, - }) - ); - assert_eq!( - statement_summary_json(&summary, &StatementTiming::NotReported("duckdb".to_owned())) - ["compile_time"], - serde_json::Value::Null - ); } diff --git a/crates/sqlbuild-core/src/progress/tests/test_types.rs b/crates/sqlbuild-core/src/progress/tests/test_types.rs index 1606615f18..ac0b995f26 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_types.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::cli_text::models::CliStyle; -use crate::progress::models::{ProgressEvent, ProgressSettings}; +use crate::progress::models::{ProgressEvent, ProgressSettings, StatementSummary, StatementTiming}; use crate::terminal_events::models::TerminalRequest; pub(super) struct ProjectionTestCase { @@ -57,3 +57,18 @@ pub(super) struct AuditRowTestCase { pub(super) recorded_outcomes: Vec<(&'static str, &'static str)>, pub(super) expected_lines: Vec<&'static str>, } + +pub(super) struct StatementCountTestCase { + pub(super) description: &'static str, + pub(super) events: Vec, + pub(super) expected_by_kind: Vec<(&'static str, u64)>, + pub(super) expected_query_ids: Vec<&'static str>, +} + +pub(super) struct StatementSummaryTextTestCase { + pub(super) description: &'static str, + pub(super) summary: StatementSummary, + pub(super) timing: StatementTiming, + pub(super) expected_line: Option<&'static str>, + pub(super) expected_json: serde_json::Value, +} diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index a76c70eb70..3ac29f7c87 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -162,6 +162,11 @@ const R_TARGET_SETTINGS: RecordSpec = RecordSpec { FieldSpec::new("schema_preserved", FieldDefault::False), ], }; +const R_PROVIDER_SETUP_CONTEXT: RecordSpec = RecordSpec { + name: "ProviderSetupContext", + doc: "What a provider's setup receives: the active target's resolved settings.", + fields: &[FieldSpec::new("target_config", FieldDefault::None)], +}; const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { name: "LocalTargetConfig", doc: "Local developer overrides for one named target.", @@ -742,6 +747,7 @@ pub(crate) const CONTRACT_RECORDS: &[RecordSpec] = &[ R_TARGET_WAREHOUSES_CONFIG, R_TARGET_CONFIG, R_TARGET_SETTINGS, + R_PROVIDER_SETUP_CONTEXT, R_LOCAL_TARGET_CONFIG, R_SETTINGS_CONFIG, R_SCOPES_CONFIG, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs index 69565744d3..3a3a2c281d 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/native_classes/ids.rs @@ -398,6 +398,7 @@ native_id_names! { RecordId::Project => "Project", RecordId::ProjectConfig => "ProjectConfig", RecordId::ProjectPath => "ProjectPath", + RecordId::ProviderSetupContext => "ProviderSetupContext", RecordId::PythonHookEntry => "PythonHookEntry", RecordId::PythonIdentityDependency => "PythonIdentityDependency", RecordId::PythonIdentityStatus => "PythonIdentityStatus", diff --git a/crates/sqlbuild-python-base/src/records/types.rs b/crates/sqlbuild-python-base/src/records/types.rs index ef1b92a365..8bbe5811e5 100644 --- a/crates/sqlbuild-python-base/src/records/types.rs +++ b/crates/sqlbuild-python-base/src/records/types.rs @@ -391,6 +391,7 @@ pub enum RecordId { Project, ProjectConfig, ProjectPath, + ProviderSetupContext, PythonHookEntry, PythonIdentityDependency, PythonIdentityStatus, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 8e3408df42..0161c6e24f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,6 +19,9 @@ use crate::bindings::_helpers::build_command::planning::{ write_compile_target, write_plan_text, }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, selected_target_settings, +}; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::observations::source_capture::{ @@ -32,6 +35,7 @@ use sqlbuild_python_base::build_command::main::tuple::tuple; use sqlbuild_python_base::diagnostics::models::BuildPhaseTimingTracker; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; @@ -39,12 +43,31 @@ use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { let py: Python<'_> = context.py(); + let discovered: Bound<'_, PyAny> = context.get("discovered_inputs")?; + let selected_target: Bound<'_, PyAny> = native_field(&context.request, "selected_target")?; + let (_, effective_vars, _): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = + build_effective_runtime_config( + &discovered, + Some(selected_target.clone()).filter(|value| !value.is_none()), + Some(native_field(&context.request, "cli_vars")?).filter(|value| !value.is_none()), + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? + .extract()?; + let target_config: TargetSettingsValues = selected_target_settings( + (&discovered, &selected_target), + &context.get("adapter_name")?.extract::()?, + ( + &context.get("connection_config")?, + &effective_vars, + &context.get("adapter")?, + ), + )?; let session: Bound<'_, PyAny> = call_with( &host_entry(py, UserCodeHost::BuildProviderSession)?, - &[( - "discovered_providers", - context.get("discovered_inputs")?.getattr("providers")?, - )], + &[ + ("discovered_providers", discovered.getattr("providers")?), + ("setup_context", target_config.setup_context(py)?), + ], )?; let built: PyResult = run_planned_build(context, &native_field(&session, "providers")?); let closed: PyResult> = session.call_method0("close"); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index a3332b007c..66dbbfc4a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -685,10 +685,13 @@ pub(crate) fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P require_used_secrets(&discovered, &uses)?; let session: Bound<'_, PyAny> = call_with( &host_entry(py, UserCodeHost::BuildProviderSession)?, - &[( - "discovered_providers", - native_field(&invocation, "discovered_inputs")?.getattr("providers")?, - )], + &[ + ( + "discovered_providers", + native_field(&invocation, "discovered_inputs")?.getattr("providers")?, + ), + ("setup_context", prepared.target_config.setup_context(py)?), + ], )?; let outcome: PyResult = (|| { let results: Vec> = execute( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index d46aa32b88..f797a8508e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -6,7 +6,7 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods}; +use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::target_checks::constants::{ @@ -44,6 +44,8 @@ const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; const NAMED_TARGET_HELP_PURPOSE: &str = "to build this target into that schema"; use crate::bindings::_helpers::compile_attachment::target::resolved_target; use pyo3::IntoPyObjectExt; + +use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -450,9 +452,18 @@ impl TargetSettingsValues { ], ) } + + /// The `ctx` a provider's `setup` receives in a command with these settings. + pub(crate) fn setup_context<'py>(&self, py: Python<'py>) -> PyResult> { + new_record_with( + py, + RecordId::ProviderSetupContext, + &[("target_config", self.record(py)?)], + ) + } } -/// The target `target_name` with its resolved `database` and `schema`, and the schema a managed source without its own loads into. +/// The target `target_name` with its resolved locations; `loader_schema` is `None` when it resolves only inside a model, such as a `${CTX:...}` schema. pub(crate) fn target_settings<'py>( (discovered, target_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (database, schema): (Option, Option), @@ -474,8 +485,6 @@ pub(crate) fn target_settings<'py>( )? }; let none: Bound<'py, PyAny> = py.None().into_bound(py); - // A loader location that only resolves inside a model, such as a `${CTX:...}` target schema, - // has no value here; managed loads still resolve and report it themselves. let loader_schema: Option = match managed_source_namespace( (none.clone(), none), (&target_config, connection, effective_vars), @@ -493,6 +502,30 @@ pub(crate) fn target_settings<'py>( }) } +/// The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. +pub(crate) fn selected_target_settings<'py>( + (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter_name: &str, + (connection, effective_vars, adapter): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult { + let namespace: Bound<'py, PyTuple> = effective_target_namespace( + (discovered, selected_target), + (adapter_name, connection, adapter), + )?; + target_settings( + (discovered, &namespace.get_item(0)?), + ( + namespace.get_item(2)?.extract()?, + namespace.get_item(3)?.extract()?, + ), + (connection, effective_vars, adapter), + ) +} + /// `target_settings` of a compiled project's target. pub(crate) fn project_target_settings<'py>( discovered: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index b44b218eb7..bdd96f08c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -21,9 +21,10 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::compile_attachment::target::effective_target_namespace; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compile_driver::targets::{TargetSettingsValues, target_settings}; +use crate::bindings::_helpers::compile_driver::targets::{ + TargetSettingsValues, selected_target_settings, +}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -84,30 +85,6 @@ struct Preparation<'py> { target_config: TargetSettingsValues, } -/// The selected target's settings, with its database and schema resolved as `sqb debug` resolves them. -fn selected_target_settings<'py>( - (discovered, selected_target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - adapter_name: &str, - (connection, effective_vars, adapter): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), -) -> PyResult { - let namespace: Bound<'py, PyTuple> = effective_target_namespace( - (discovered, selected_target), - (adapter_name, connection, adapter), - )?; - target_settings( - (discovered, &namespace.get_item(0)?), - ( - namespace.get_item(2)?.extract()?, - namespace.get_item(3)?.extract()?, - ), - (connection, effective_vars, adapter), - ) -} - fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } @@ -381,18 +358,21 @@ fn prepare<'py>( } else { new_record(py, RecordId::CursorOverrides, (), None)? }; - let provider_session: Bound<'py, PyAny> = call_with( - &host_entry(py, UserCodeHost::BuildProviderSession)?, - &[( - "discovered_providers", - native_field(discovered, "providers")?, - )], - )?; let target_config: TargetSettingsValues = selected_target_settings( (discovered, &native_field(request, "selected_target")?), &adapter_name.extract::()?, (&connection_config, &runtime.1, &adapter), )?; + let provider_session: Bound<'py, PyAny> = call_with( + &host_entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ( + "discovered_providers", + native_field(discovered, "providers")?, + ), + ("setup_context", target_config.setup_context(py)?), + ], + )?; Ok(Preparation { target_config, adapter_name: adapter_name.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index a3c45ce10a..73918873fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -5,4 +5,5 @@ pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod resource_costs_state; +pub(crate) mod statement_report; pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs rename to crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs index 7bce151b5c..0a163de366 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/statements.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs @@ -41,6 +41,22 @@ pub(crate) struct StatementReport { } impl StatementReport { + /// The report of a `sqb test` invocation's adapter and connection. + pub(crate) fn of_invocation( + invocation: &Bound<'_, PyAny>, + started: Instant, + ) -> PyResult> { + Self::read( + invocation.py(), + &StatementSource { + adapter_name: &invocation.getattr("adapter_name")?.extract::()?, + adapter: &invocation.getattr("adapter")?, + connection_config: &invocation.getattr("connection_config")?, + started, + }, + ) + } + /// The report of the installed projector, or nothing when no projector is installed. pub(crate) fn read(py: Python<'_>, source: &StatementSource<'_, '_>) -> PyResult> { let Some(projector) = current_projector(py)? else { @@ -58,9 +74,8 @@ impl StatementReport { /// `payload` with the `statements` section added. pub(crate) fn with_section(&self, payload: &str) -> PyResult { - let invalid = |error: serde_json::Error| { - pyo3::exceptions::PyValueError::new_err(error.to_string()) - }; + let invalid = + |error: serde_json::Error| pyo3::exceptions::PyValueError::new_err(error.to_string()); let mut document: Value = serde_json::from_str(payload).map_err(invalid)?; if let Value::Object(fields) = &mut document { fields.insert( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index a4c1050e95..2a86603b89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -9,5 +9,4 @@ pub(crate) mod integration_records; pub(crate) mod projection; pub(crate) mod registration; pub(crate) mod resource_ids; -pub(crate) mod statements; pub(crate) mod terminal_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index f57a68e517..197f8556d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -24,9 +24,9 @@ use crate::bindings::_helpers::change_selection::base_snapshot::{ use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, with_since_section, }; +use crate::bindings::_helpers::native_host::statement_report::{StatementReport, StatementSource}; use crate::bindings::_helpers::result_output::documents::format_scenario_execution_json_with; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; -use crate::bindings::_helpers::result_output::statements::{StatementReport, StatementSource}; use crate::bindings::_helpers::scenario_command::conversions::scenario_result_row; use crate::bindings::_helpers::scenario_command::presentation::{ CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index d78e818b4c..deca2b06ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -43,10 +43,10 @@ use crate::bindings::_helpers::compile_driver::static_context::compile_static_co use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::native_host::statement_report::StatementReport; use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; -use crate::bindings::_helpers::result_output::statements::{StatementReport, StatementSource}; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; use crate::bindings::_helpers::sql_test_command::python_tests::{ @@ -678,26 +678,12 @@ fn complete( let style: CliStyle = CliStyle { use_color: invocation.getattr("use_color")?.is_truthy()?, }; - stream.call_method1( - "write", - (render_test_completion( - (passed, results.len()), - &failures, - style, - ),), - )?; - let report: Option = StatementReport::read( - py, - &StatementSource { - adapter_name: &invocation.getattr("adapter_name")?.extract::()?, - adapter: &invocation.getattr("adapter")?, - connection_config: &invocation.getattr("connection_config")?, - started, - }, - )?; + let report: Option = StatementReport::of_invocation(invocation, started)?; + let mut completion: String = render_test_completion((passed, results.len()), &failures, style); if let Some(line) = report.as_ref().and_then(StatementReport::line) { - stream.call_method1("write", (format!("\n{line}\n"),))?; + completion.push_str(&format!("\n{line}\n")); } + stream.call_method1("write", (completion,))?; stream.call_method0("flush")?; let payload: String = with_since_section(test_execution_json(py, results)?, since_json)?; Ok(( @@ -914,7 +900,8 @@ fn run_sql_tests<'py>( &pipeline.getattr("plan_output")?, &results, )?; - let (code, payload): (i32, String) = complete(invocation, (&results, since_json.as_deref()), started)?; + let (code, payload): (i32, String) = + complete(invocation, (&results, since_json.as_deref()), started)?; Ok((code, python_tests, Some(payload))) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index 827a3b7360..c9217cb9a4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -171,6 +171,23 @@ class WarehouseClient(Provider): Both `setup` and `teardown` are optional. A provider with only field declarations and no lifecycle methods is valid; it acts as a validated configuration object. +### The setup context + +In `sqb build`, `sqb check` and `sqb load`, `setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](overview.md#target-settings)): + +```python +class PartnerStage(Provider): + """Stage for partner exports.""" + + def setup(self, ctx): + target = ctx.target_config + self.stage = f"{target.database}.{target.loader_schema}.partner_stage" +``` + +Details: + +- A provider first used by an event exporter or command output sink, or by an orchestration sensor or calendar, is set up with `ctx` set to `None`. Check for `None` when such a provider also reads the target. + ## Configuration from environment variables Because `Provider` extends `pydantic-settings BaseSettings`, fields without defaults are read from environment variables. The environment variable name matches the field name in uppercase: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1c3b261fed..11632aced0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -661,6 +661,12 @@ class NativeSqlTestArtifact(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> NativeSqlTestArtifact: ... +class ProviderSetupContext(NativeRecord): + """What a provider's setup receives: the active target's resolved settings.""" + + target_config: TargetSettings | None + def __init__(self, target_config: TargetSettings | None = ...) -> None: ... + class RelationFixtureDiagnostic(NativeRecord): """One statically provable invalid fixture condition.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index a8eb67bede..b20aad918f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1642,3 +1642,13 @@ class PythonTargetConfigBuildE2ETestCase: project_target_lines: str local_target_lines: str expected_settings: dict[str, object] + + +@dataclass(frozen=True) +class CtxTargetSchemaE2ETestCase: + """Test case for commands under a target schema that resolves only inside a model.""" + + description: str + target_schema: str + expected_model_schema: str + expected_loader_schema: str | None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index cdeee5aeec..e9178ec58a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -13,6 +13,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( BuildE2ETestCase, + CtxTargetSchemaE2ETestCase, DeferCloneBuildE2ETestCase, DirectChangesOnlyBuildE2ETestCase, DirectPythonBuildHardeningE2ETestCase, @@ -1867,30 +1868,41 @@ def test_given_local_target_override_when_running_python_then_every_context_sees "[targets.dev]\n" + test_case.project_target_lines ), "python/recorder.py": _TARGET_CONFIG_RECORDER, + "providers/target_probe.py": ( + "from python.recorder import record\n" + "from sqlbuild.providers import Provider\n\n" + "class TargetProbe(Provider):\n" + " '''Test provider TargetProbe.'''\n\n" + " def setup(self, ctx):\n" + " record(ctx, 'provider')\n" + ), "python/tasks/configured.py": ( + "from providers.target_probe import TargetProbe\n" "from python.recorder import record\n" "from sqlbuild.tasks import task\n\n" "@task\n" - "def configured_task(ctx):\n" + "def configured_task(ctx, target_probe: TargetProbe):\n" " '''Test task configured_task.'''\n" " record(ctx, 'task')\n" " return ctx.result()\n" ), "python/loaders/raw.py": ( + "from providers.target_probe import TargetProbe\n" "from python.recorder import record\n" "from sqlbuild.loaders import loader\n\n" "@loader\n" - "def raw_orders(ctx):\n" + "def raw_orders(ctx, target_probe: TargetProbe):\n" " '''Test loader raw_orders.'''\n" " record(ctx, 'loader')\n" " return [{'order_id': 1}]\n" ), "python/checks/configured.py": ( + "from providers.target_probe import TargetProbe\n" "from python.recorder import record\n" "from python.tasks.configured import configured_task\n" "from sqlbuild.checks import check\n\n" "@check(depends_on=configured_task)\n" - "def configured_check(ctx):\n" + "def configured_check(ctx, target_probe: TargetProbe):\n" " '''Test check configured_check.'''\n" " record(ctx, 'check')\n" " return ctx.pass_(message='recorded')\n" @@ -1925,14 +1937,36 @@ def test_given_local_target_override_when_running_python_then_every_context_sees json.loads(line) for line in (project_dir / "target_config.jsonl").read_text(encoding="utf-8").splitlines() ] - assert {str(line.pop("node")) for line in recorded} == {"task", "loader", "check"} + # Each command (build, load, check) sets the provider up once, before its first node runs. + assert [str(line.pop("node")) for line in recorded] == [ + "provider", + "task", + "loader", + "check", + "provider", + "loader", + "provider", + "check", + ] assert all(line == test_case.expected_settings for line in recorded), recorded +@pytest.mark.parametrize( + "test_case", + [ + CtxTargetSchemaE2ETestCase( + description="a per-model schema has no loader location and fails no command", + target_schema="s_${CTX:model.name}", + expected_model_schema="s_orders", + expected_loader_schema=None, + ) + ], + ids=lambda case: case.description, +) def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_loader_schema( + test_case: CtxTargetSchemaE2ETestCase, tmp_path: Path, ) -> None: - # A per-model schema has no single loader location outside a model; it must not fail commands. project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="ctx_target_project", @@ -1944,7 +1978,7 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load "[connection]\n" 'database = "ctx_target_project.duckdb"\n\n' "[targets.dev]\n" - 'schema = "s_${CTX:model.name}"\n' + f'schema = "{test_case.target_schema}"\n' ), "models/orders.sql": "MODEL (description 'Test model orders.');\n\nSELECT 1 AS order_id\n", "python/recorder.py": _TARGET_CONFIG_RECORDER, @@ -1978,14 +2012,16 @@ def test_given_ctx_templated_target_schema_when_running_then_python_sees_no_load for result in results: assert result.returncode == 0, result.stdout + result.stderr db_path: Path = project_dir / "ctx_target_project.duckdb" - assert table_exists(db_path=db_path, table_name="orders", schema="s_orders") + assert table_exists( + db_path=db_path, table_name="orders", schema=test_case.expected_model_schema + ) assert table_exists(db_path=db_path, table_name="raw_orders", schema="raw") recorded: list[dict[str, object]] = [ json.loads(line) for line in (project_dir / "target_config.jsonl").read_text(encoding="utf-8").splitlines() ] assert [(line["node"], line["name"], line["loader_schema"]) for line in recorded] == [ - ("loader", "dev", None) + ("loader", "dev", test_case.expected_loader_schema) ], recorded diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 7b37ec6cb3..1a1e72bcf6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -649,3 +649,12 @@ class StatementSummaryE2ETestCase: expected_exit_code: int expected_by_kind: Mapping[str, int] expected_line: str + + +@dataclass(frozen=True) +class ScenarioStatementSummaryE2ETestCase: + """A `sqb scenario test --json` run whose statement summary must match its progress line.""" + + description: str + expected_exit_code: int + expected_kinds: frozenset[str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index eea08cac2f..bfa49fbea8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -2477,3 +2477,13 @@ def prepare_since_repository(*, tmp_path: Path, extra_files: Mapping[str, str]) ) commit_project_on_main(repository=project) return project + + +def statement_summary_test_file(name: str, doubled: int) -> str: + """Return a SQL test of the `orders` model expecting `doubled` for an amount of 5.""" + + return ( + f'TEST (name "{name}");\n\nWITH\n' + "__source__raw_orders AS (\n SELECT 1 AS order_id, 5 AS amount\n),\n" + f"__expected__orders AS (\n SELECT 1 AS order_id, {doubled} AS doubled\n)\nSELECT 1\n" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py index f33139643d..3c3da37a76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_summary.py @@ -10,8 +10,10 @@ build_scenario_project_files, ) from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + ScenarioStatementSummaryE2ETestCase, StatementSummaryE2ETestCase, ) +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import statement_summary_test_file from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb _PROJECT_FILES: dict[str, str] = { @@ -33,14 +35,6 @@ } -def _test_file(name: str, doubled: int) -> str: - return ( - f'TEST (name "{name}");\n\nWITH\n' - "__source__raw_orders AS (\n SELECT 1 AS order_id, 5 AS amount\n),\n" - f"__expected__orders AS (\n SELECT 1 AS order_id, {doubled} AS doubled\n)\nSELECT 1\n" - ) - - @pytest.mark.parametrize( "test_case", ( @@ -73,7 +67,9 @@ def test_given_sql_tests_when_running_then_statement_summary_counts_each_stateme ) -> None: files: dict[str, str] = dict(_PROJECT_FILES) for index, doubled in enumerate(test_case.expected_doubled): - files[f"tests/unit/test_orders_{index}.sql"] = _test_file(f"orders_{index}", doubled) + files[f"tests/unit/test_orders_{index}.sql"] = statement_summary_test_file( + f"orders_{index}", doubled + ) project: Path = prepare_inline_project( tmp_path=tmp_path, project_name="statement_summary", repo_files=files ) @@ -94,8 +90,19 @@ def test_given_sql_tests_when_running_then_statement_summary_counts_each_stateme assert test_case.expected_line in json_run.stderr.splitlines(), json_run.stderr +@pytest.mark.parametrize( + "test_case", + ( + ScenarioStatementSummaryE2ETestCase( + description="passing and failing scenarios report their statements", + expected_exit_code=1, + expected_kinds=frozenset({"CREATE", "DROP"}), + ), + ), + ids=lambda case: case.description, +) def test_given_scenarios_when_running_with_json_then_statement_summary_matches_progress_line( - tmp_path: Path, + tmp_path: Path, test_case: ScenarioStatementSummaryE2ETestCase ) -> None: project: Path = prepare_inline_project( tmp_path=tmp_path, @@ -107,10 +114,11 @@ def test_given_scenarios_when_running_with_json_then_statement_summary_matches_p command=("--no-color", "scenario", "test", "--json"), project_dir=project ) - assert result.returncode == 1, result.stderr + assert result.returncode == test_case.expected_exit_code, result.stderr statements: dict[str, object] = json.loads(result.stdout)["statements"] by_kind: dict[str, int] = statements["by_kind"] # type: ignore[assignment] - assert statements["total"] == sum(by_kind.values()) and {"CREATE", "DROP"} <= set(by_kind) + assert statements["total"] == sum(by_kind.values()), statements + assert test_case.expected_kinds <= set(by_kind), statements assert statements["compile_time"] is None kinds: str = ", ".join( f"{kind} {count}" diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index b9b6efd791..3f6481302b 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -153,6 +153,23 @@ class WarehouseClient(Provider): Both `setup` and `teardown` are optional. A provider with only field declarations and no lifecycle methods is valid; it acts as a validated configuration object. +### The setup context + +In `sqb build`, `sqb check` and `sqb load`, `setup(ctx)` receives the active target's resolved settings as `ctx.target_config`, the same values Python nodes see (see [Target settings](/docs/concepts/python-nodes/overview#target-settings)): + +```python +class PartnerStage(Provider): + """Stage for partner exports.""" + + def setup(self, ctx): + target = ctx.target_config + self.stage = f"{target.database}.{target.loader_schema}.partner_stage" +``` + +Details: + +- A provider first used by an event exporter or command output sink, or by an orchestration sensor or calendar, is set up with `ctx` set to `None`. Check for `None` when such a provider also reads the target. + ## Configuration from environment variables Because `Provider` extends `pydantic-settings BaseSettings`, fields without defaults are read from environment variables. The environment variable name matches the field name in uppercase: From 93b70ea641121e578823ae9c966159ac2dc80d13 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:21:34 +0000 Subject: [PATCH 12/15] fix: print the sqb test results before reading statement history The pass/fail summary and failures are written and flushed first, so a slow or interrupted warehouse history read cannot delay or hide them. --- .../bindings/_helpers/sql_test_command/test_command.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 4fcdf221bb..91a8fbf00a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -58,6 +58,7 @@ use crate::bindings::_helpers::sql_test_command::timeouts::{ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::emit::emit; use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -679,13 +680,12 @@ fn complete( let style: CliStyle = CliStyle { use_color: invocation.getattr("use_color")?.is_truthy()?, }; + let completion: String = render_test_completion((passed, results.len()), &failures, style); + emit(&stream, &completion)?; let report: Option = StatementReport::of_invocation(invocation, started)?; - let mut completion: String = render_test_completion((passed, results.len()), &failures, style); if let Some(line) = report.as_ref().and_then(StatementReport::line) { - completion.push_str(&format!("\n{line}\n")); + emit(&stream, &format!("\n{line}\n"))?; } - stream.call_method1("write", (completion,))?; - stream.call_method0("flush")?; let payload: String = with_since_section(test_execution_json(py, results)?, since_json)?; Ok(( test_exit_code(passed, results.len()), From 35a7c8aa40fc51ab68824c3e52ef5619dc0c3c0d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:29:45 +0000 Subject: [PATCH 13/15] chore: set the Python bindings fensu budgets to their measured values --- fensu.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fensu.toml b/fensu.toml index cc1be40e58..fa5afba470 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1672,7 +1672,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147491 } +thresholds = { max_crate_code_lines = 147801 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -2731,7 +2731,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=99", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=97", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", From 10b2e057911baafe9ff566c4e9e52577d79e8a6e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:09:54 +0000 Subject: [PATCH 14/15] refactor: share one JSON section helper between the since and statements sections Both sqb test --json sections are now added by sqlbuild-core's json with_section. The Python bindings cap follows the smaller crate. --- crates/sqlbuild-core/src/json/main/mod.rs | 1 + .../src/json/main/with_section.rs | 16 ++++++++++++++ .../_helpers/change_selection/selection.rs | 19 +++++++---------- .../_helpers/native_host/statement_report.rs | 21 +++++++------------ fensu.toml | 2 +- 5 files changed, 32 insertions(+), 27 deletions(-) create mode 100644 crates/sqlbuild-core/src/json/main/with_section.rs diff --git a/crates/sqlbuild-core/src/json/main/mod.rs b/crates/sqlbuild-core/src/json/main/mod.rs index c295400d51..18b8ecee33 100644 --- a/crates/sqlbuild-core/src/json/main/mod.rs +++ b/crates/sqlbuild-core/src/json/main/mod.rs @@ -1,2 +1,3 @@ pub mod dumps; pub mod python_float_repr; +pub mod with_section; diff --git a/crates/sqlbuild-core/src/json/main/with_section.rs b/crates/sqlbuild-core/src/json/main/with_section.rs new file mode 100644 index 0000000000..9a818f4ef8 --- /dev/null +++ b/crates/sqlbuild-core/src/json/main/with_section.rs @@ -0,0 +1,16 @@ +//! Add a named section to a JSON document. + +use serde_json::Value; + +/// `payload`, a JSON document, with `section` added under `name`, pretty-printed with a trailing newline. +pub fn with_section( + payload: &str, + name: &str, + section: Value, +) -> Result { + let mut document: Value = serde_json::from_str(payload)?; + if let Value::Object(fields) = &mut document { + fields.insert(name.to_owned(), section); + } + Ok(format!("{}\n", serde_json::to_string_pretty(&document)?)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 293c886880..9d570e4062 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -6,7 +6,6 @@ use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; -use serde_json::Value; use sqlbuild_analysis::definition_changes::constants::{ MACRO_KIND, MODEL_KIND, SCENARIO_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, TEST_FILE_KIND, @@ -20,6 +19,7 @@ use sqlbuild_analysis::definition_changes::main::selection_report::selection_rep use sqlbuild_analysis::definition_changes::models::{ DefinitionChanges, DefinitionSnapshot, ExecutedItem, SelectedItem, }; +use sqlbuild_core::json::main::with_section::with_section; use crate::bindings::_helpers::change_selection::base_snapshot::BaseComparison; use crate::bindings::_helpers::change_selection::definition_snapshot::{ @@ -311,17 +311,12 @@ pub(crate) fn with_since_section(payload: String, since: Option<&str>) -> PyResu return Ok(payload); }; let invalid = |error: serde_json::Error| PyValueError::new_err(error.to_string()); - let mut document: Value = serde_json::from_str(&payload).map_err(invalid)?; - if let Value::Object(fields) = &mut document { - fields.insert( - SINCE_SECTION.to_owned(), - serde_json::from_str(since).map_err(invalid)?, - ); - } - Ok(format!( - "{}\n", - serde_json::to_string_pretty(&document).map_err(invalid)? - )) + with_section( + &payload, + SINCE_SECTION, + serde_json::from_str(since).map_err(invalid)?, + ) + .map_err(invalid) } /// The project's compiled tests and models by name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs index 0a163de366..f34809d85d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs @@ -5,8 +5,8 @@ use std::time::Instant; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTypeMethods}; -use serde_json::Value; use sqlbuild_core::cost_allocation::main::statement_timings_sql::statement_timings_sql; +use sqlbuild_core::json::main::with_section::with_section; use sqlbuild_core::progress::main::format_statement_summary::format_statement_summary; use sqlbuild_core::progress::main::statement_summary_json::statement_summary_json; use sqlbuild_core::progress::models::{StatementSummary, StatementTiming, StatementTimings}; @@ -74,19 +74,12 @@ impl StatementReport { /// `payload` with the `statements` section added. pub(crate) fn with_section(&self, payload: &str) -> PyResult { - let invalid = - |error: serde_json::Error| pyo3::exceptions::PyValueError::new_err(error.to_string()); - let mut document: Value = serde_json::from_str(payload).map_err(invalid)?; - if let Value::Object(fields) = &mut document { - fields.insert( - STATEMENTS_SECTION.to_owned(), - statement_summary_json(&self.summary, &self.timing), - ); - } - Ok(format!( - "{}\n", - serde_json::to_string_pretty(&document).map_err(invalid)? - )) + with_section( + payload, + STATEMENTS_SECTION, + statement_summary_json(&self.summary, &self.timing), + ) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string())) } } diff --git a/fensu.toml b/fensu.toml index 1c9aa162b9..c4f6c000b9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1670,7 +1670,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147801 } +thresholds = { max_crate_code_lines = 147789 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] From ae781cdc5421107e7b0ffa8edd794ca975d51223 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:16:30 +0000 Subject: [PATCH 15/15] test: allow table-name rows in the worker execution cases --- .../compiler/compiled_project/publication/_test_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index c480609117..78536260c1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -111,7 +111,7 @@ class WorkerExecutionTestCase: model_sql: str warehouse_query: str expected_status: str - expected_rows: list[tuple[int]] + expected_rows: list[tuple[int] | tuple[str]] expected_node_count: int = 1 # Node counts by status when the nodes don't all share the run's status. expected_node_counts: dict[str, int] | None = None