From e42d4bfb14d69e145f9a30ac3e5aade0172ed08e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:44:17 +0000 Subject: [PATCH 1/3] fix(seed): load seeds from a shared queue so a slow seed holds up only itself sqb seed assigned seeds to its connections round-robin up front, so the seeds behind a slow one on the same connection waited for it while other connections sat idle. Each connection now takes the next seed from one queue. Builds already schedule seeds from a shared ready queue. --- .../executor/pipeline/_helpers/seeding.py | 23 ++++++++----- .../executor/pipeline/_helpers/_test_types.py | 11 ++++++ .../executor/pipeline/_helpers/helpers.py | 34 +++++++++++++++++++ .../pipeline/_helpers/test_seeding.py | 33 ++++++++++++++++++ 4 files changed, 92 insertions(+), 9 deletions(-) diff --git a/src/sqlbuild/executor/pipeline/_helpers/seeding.py b/src/sqlbuild/executor/pipeline/_helpers/seeding.py index 71bfa9083f..b485db73b3 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/seeding.py +++ b/src/sqlbuild/executor/pipeline/_helpers/seeding.py @@ -2,6 +2,7 @@ from __future__ import annotations +import queue import time from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor @@ -59,6 +60,10 @@ def run_seed_pipeline( if on_connection_complete is not None: on_connection_complete(effective_concurrency, elapsed_seconds=time.monotonic() - start) + pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]] = queue.SimpleQueue() + indexed_entry: tuple[int, SeedPlanEntry] + for indexed_entry in enumerate(seed_entries): + pending.put(indexed_entry) try: with ThreadPoolExecutor(max_workers=effective_concurrency) as pool: futures: tuple[Future[tuple[tuple[int, SeedExecutionResult], ...]], ...] = tuple( @@ -67,9 +72,7 @@ def run_seed_pipeline( pool.submit( copy_context().run, _run_seed_worker, - worker_index=worker_index, - effective_concurrency=effective_concurrency, - seed_entries=seed_entries, + pending=pending, adapter=adapter, connection=connections[worker_index], run_id=run_id, @@ -97,21 +100,23 @@ def run_seed_pipeline( def _run_seed_worker( *, - worker_index: int, - effective_concurrency: int, - seed_entries: tuple[SeedPlanEntry, ...], + pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]], adapter: BaseAdapter, connection: Any, run_id: str, query_change_tracking: bool, on_seed_complete: Callable[[SeedExecutionResult], None] | None, ) -> tuple[tuple[int, SeedExecutionResult], ...]: + """Load seeds from the shared queue until it is empty, so a slow seed holds up only itself.""" + indexed_results: list[tuple[int, SeedExecutionResult]] = [] seed_index: int entry: SeedPlanEntry - for seed_index, entry in enumerate(seed_entries): - if seed_index % effective_concurrency != worker_index: - continue + while True: + try: + seed_index, entry = pending.get_nowait() + except queue.Empty: + break with ResourceAttemptLifecycle( resource_id=f"seed:{entry.name}", resource_kind="seed", diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py index 7853c224f1..69b846bda2 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py @@ -139,6 +139,17 @@ class ScenarioPlanningLifecycleTestCase: expected_terminal: str +@dataclass(frozen=True) +class SeedPipelineQueueTestCase: + """Seeds behind a slow one: which load before it finishes.""" + + description: str + seed_names: tuple[str, ...] + slow_seed: str + max_concurrency: int + expected_load_order: tuple[str, ...] + + @dataclass(frozen=True) class SeedPipelineConcurrencyTestCase: """One seed pipeline concurrency orchestration case.""" diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index a25935e76d..16ff0549c2 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -186,6 +186,40 @@ def load_seed( self.loads.append((destination, connection)) +def _no_signal() -> None: + """The slow seed signals nothing when it finishes.""" + + +class SlowSeedPipelineAdapter(SeedPipelineTestAdapter): + """Adapter whose slow seed finishes only after every other seed has loaded.""" + + def __init__(self, *, slow_target: str, other_count: int) -> None: + super().__init__(barrier_targets=(slow_target,)) + self.slow_target: str = slow_target + self.other_count: int = other_count + self.others_done: threading.Semaphore = threading.Semaphore(0) + self.order: list[str] = [] + + def load_seed( + self, + connection: Any, + *, + destination: str, + file_path: Path, + columns: tuple[ColumnInfo, ...], + csv_settings: SeedCsvSettings = DEFAULT_SEED_CSV_SETTINGS, + replace: bool = True, + infer_types: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + del connection, file_path, columns, csv_settings, replace, infer_types, statement_recorder + waits: int = self.other_count * (destination == self.slow_target) + released: bool = all(self.others_done.acquire(timeout=5) for _ in range(waits)) + self.order.append(destination + " timed out" * (not released)) + finish: dict[bool, Callable[[], None]] = {True: _no_signal, False: self.others_done.release} + finish[destination == self.slow_target]() + + class ScenarioPipelinePlanBuilder: """Callable plan builder that can fail for one configured scenario.""" diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py index 28c6ee74a1..c1ba05633d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py @@ -20,9 +20,11 @@ from tests.unit.src.sqlbuild.executor.pipeline._helpers._test_types import ( SeedPipelineConcurrencyTestCase, SeedPipelineLifecycleTestCase, + SeedPipelineQueueTestCase, ) from tests.unit.src.sqlbuild.executor.pipeline._helpers.helpers import ( SeedPipelineTestAdapter, + SlowSeedPipelineAdapter, build_seed_plan, ) @@ -166,3 +168,34 @@ def test_given_seed_executor_raises_when_running_worker_then_attempt_fails_once_ assert tuple(event.event_type for event in resource_events) == test_case.expected_terminal_types assert {event.resource_id for event in resource_events} == test_case.expected_resource_ids assert callbacks == [] + + +@pytest.mark.parametrize( + "test_case", + [ + SeedPipelineQueueTestCase( + description="seeds behind a slow one load on the free connection meanwhile", + seed_names=("seed_slow", "seed_b", "seed_c", "seed_d"), + slow_seed="seed_slow", + max_concurrency=2, + expected_load_order=("seed_b", "seed_c", "seed_d", "seed_slow"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_slow_seed_when_running_seed_pipeline_then_other_seeds_take_the_free_connection( + test_case: SeedPipelineQueueTestCase, +) -> None: + adapter: SlowSeedPipelineAdapter = SlowSeedPipelineAdapter( + slow_target=test_case.slow_seed, other_count=len(test_case.seed_names) - 1 + ) + + results: tuple[SeedExecutionResult, ...] = run_seed_pipeline( + plan=build_seed_plan(seed_names=test_case.seed_names), + connection_config={}, + adapter=adapter, + max_concurrency=test_case.max_concurrency, + ) + + assert tuple(adapter.order) == test_case.expected_load_order, test_case.description + assert tuple(result.seed_name for result in results) == test_case.seed_names From f9f7bab5ea0794e13fffb8b990e6892b74927015 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 02:48:25 +0000 Subject: [PATCH 2/3] fix(worker): plan from the compile the worker verified instead of compiling again A worker compiled the project to verify it matches the published snapshot, then threw that compile away, and the build compiled the project a second time before planning: about 7 s of every run's start-up on a 2,280-model project, and twice that under load. The worker now hands its verified compile to the build, which plans from it as a location's plan preview does. A build whose compile could differ still compiles: one with run variables, SQL validation off or a dbt manifest to resolve references from. --- .../src/records/constants.rs | 1 + .../_helpers/build_command/planning.rs | 59 +++++++++++++++---- .../_helpers/location_command/command.rs | 4 +- .../_helpers/location_command/worker.rs | 20 +++++-- src/sqlbuild/_native.pyi | 2 + .../publication/_test_types.py | 2 + .../compiled_project/publication/helpers.py | 11 ++++ .../publication/test_worker.py | 39 ++++++++++++ 8 files changed, 118 insertions(+), 20 deletions(-) diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index 305b9ce34a..f7f2d5cde5 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -273,6 +273,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("selector_files", FieldDefault::EmptyTuple), FieldSpec::new("test_timeout", FieldDefault::None), FieldSpec::new("discovered_inputs", FieldDefault::None), + FieldSpec::new("compiled_project", FieldDefault::None), ], }; const BUILD_INVOCATION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index eb8395976e..d16b491703 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -1,6 +1,10 @@ //! Compile the build plan, show it, gate it and write the compile target. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyDictMethods, PyErr, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_config::command_connection::constants::{ + CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, +}; use sqlbuild_core::build_command::constants::{ EXECUTION_LIMIT_BUILD_NOTE, MISSING_ORIGIN_BUILD_HELP, RETENTION_DECREASE_BUILD_HELP, SNAPSHOT_FULL_REFRESH_BUILD_HELP, TABLE_TYPE_DOWNGRADE_BUILD_HELP, @@ -13,7 +17,7 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::enforce_plan_policies; use crate::bindings::_helpers::build_command::policies::model_limit; use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures; -use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; +use crate::bindings::_helpers::compile_driver::plan_pipeline::planned_project; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -27,6 +31,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::manifest::main::build_manifest::build_manifest; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_output::build_progress::main::with_verbose_inspection::with_verbose_inspection; use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; @@ -128,22 +133,50 @@ pub(crate) fn compile_build_plan<'py>(context: &BuildContext<'py>) -> PyResult = request_options(context)?; + let held: Option> = held_compile(context, &options)?; with_verbose_inspection(py, verbose, || { - run_compile_pipeline( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", context.get("discovered_inputs")?), - ("adapter", context.get("adapter")?), - ("options", request_options(context)?), - ("hooks", hooks), - ], - )?), + planned_project( + (&context.get("discovered_inputs")?, &context.get("adapter")?), + (&options, hooks), + (held.as_ref(), None), ) }) } +/// The compile a worker verified, when compiling again could not differ: no run variables, validation and cache on, no dbt. +fn held_compile<'py>( + context: &BuildContext<'py>, + options: &Bound<'py, PyAny>, +) -> PyResult>> { + let held: Bound<'py, PyAny> = native_field(&context.request, "compiled_project")?; + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + let project_config: Bound<'py, PyAny> = native_field(&discovered, "project_config")?; + let dbt_manifest: Bound<'py, PyAny> = + native_field(&native_field(&project_config, "dbt")?, "target_path")?; + if held.is_none() + || native_field(options, "cli_vars")?.is_truthy()? + || native_field(options, "no_sql_validation")?.is_truthy()? + || native_field(options, "no_cache")?.is_truthy()? + || !dbt_manifest.is_none() + { + return Ok(None); + } + let connection: Bound<'py, PyDict> = build_effective_connection_config( + &discovered, + Some(native_field(options, "selected_target")?), + None, + false, + )? + .cast_into()?; + let from_dbt_profile: bool = connection + .get_item(CONNECTION_SOURCE_KEY)? + .map(|source| source.eq(DBT_PROFILE_CONNECTION_SOURCE)) + .transpose()? + .unwrap_or(false); + Ok((!from_dbt_profile).then_some(held)) +} + /// Write the formatted plan to stdout, or stderr for debug and JSON runs. pub(crate) fn write_plan_text( context: &BuildContext<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 9985b75133..f17249275d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -345,10 +345,10 @@ pub(crate) fn run_location<'py>( arguments.value("run_id")?.extract::()?, arguments.value("claim_id")?.extract::()?, ); - let ((snapshot, blocked, _), project) = compile_location(py, &arguments, &input)?; + let ((snapshot, blocked, compiled), project) = compile_location(py, &arguments, &input)?; return execute_worker( py, - (&snapshot, project, blocked), + ((&snapshot, &compiled), project, blocked), (&arguments, &input), (&bearer, identity, recall), ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 7a88a487a1..208f2e1aee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -147,12 +147,12 @@ fn job_run_settings( Ok(settings) } -/// The frozen command's request and warehouse scope; build carries the verified inputs in its request. +/// The frozen command's request and warehouse scope; build carries the verified inputs and compiled project in its request. fn command_request<'py>( py: Python<'py>, discovered: &Bound<'py, PyAny>, arguments: &KeywordArguments<'py>, - worker: &Bound<'py, NativeOrchestrationWorker>, + (worker, compiled): (&Bound<'py, NativeOrchestrationWorker>, &Bound<'py, PyAny>), ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; @@ -230,7 +230,13 @@ fn command_request<'py>( } let mut request = Invocation::new(&args)?.value(route.arguments[0].source)?; if command == RunCommand::Build { - request = replaced(&request, &[("discovered_inputs", discovered.clone())])?; + request = replaced( + &request, + &[ + ("discovered_inputs", discovered.clone()), + ("compiled_project", compiled.clone()), + ], + )?; } Ok((request, warehouse_scope(&args)?)) } @@ -343,7 +349,11 @@ fn record_compiled_vars<'py>( pub(super) fn execute_worker<'py>( py: Python<'py>, - (snapshot, project, blocked): (&NativeOrchestrationSnapshot, ProjectAdapter<'py>, NodeKeys), + ((snapshot, compiled), project, blocked): ( + (&NativeOrchestrationSnapshot, &Bound<'py, PyAny>), + ProjectAdapter<'py>, + NodeKeys, + ), (arguments, input): (&KeywordArguments<'py>, &SnapshotInput), (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), ) -> PyResult { @@ -385,7 +395,7 @@ pub(super) fn execute_worker<'py>( return Err(PyValueError::new_err(reason)); } record_compiled_vars(py, (&owner, &frozen), (arguments, input, &project))?; - let (request, scope) = command_request(py, &discovered, arguments, &owner)?; + let (request, scope) = command_request(py, &discovered, arguments, (&owner, compiled))?; write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?; let _rules = RulesAtLoadScope::enter(); within(&scope, |_| { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index de71fe3845..60ae0b7736 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6595,6 +6595,7 @@ class BuildCommandRequest(NativeRecord): selector_files: tuple[SelectorFileSummary, ...] test_timeout: str | None discovered_inputs: DiscoveredProjectInputs | None + compiled_project: object | None def __init__( self, project_dir: Path | None = None, @@ -6637,6 +6638,7 @@ class BuildCommandRequest(NativeRecord): selector_files: tuple[SelectorFileSummary, ...] = (), test_timeout: str | None = None, discovered_inputs: DiscoveredProjectInputs | None = None, + compiled_project: object | None = None, ) -> None: ... class CloneCommandRequest(NativeRecord): diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index dfeb9594bc..4d306ad30b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -142,6 +142,8 @@ class WorkerExecutionTestCase: expected_run_nodes: list[tuple[str, str, str | None, str | None, str | None]] = field( default_factory=list ) + # Whether the build compiles again instead of planning from the compile the worker verified. + expected_compiles_again: bool = False @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index a2ed27e6c1..318655288f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1037,6 +1037,17 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - ).fetchall() assert contexts, log.read_text() assert all(context is not None for (context,) in contexts) + printed: str = "".join( + str(data) + for (data,) in control.execute( + "SELECT data FROM recall_output WHERE invocation_id LIKE 'worker/%' " + "ORDER BY occurred_at, sequence" + ).fetchall() + ) + assert ("Planning from the published compiled project." not in printed) == ( + test_case.expected_compiles_again + ), printed + assert ("Compiling project..." in printed) == test_case.expected_compiles_again, printed expected_run_ids: set[str] = { run_id for (run_id,) in control.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 59b3f4edd6..64da65119a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -857,6 +857,45 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "run_tests = false\n" ), }, + expected_compiles_again=True, + ), + WorkerExecutionTestCase( + "a job with SQL validation off compiles again instead of reusing the verified compile", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + files={ + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n" + "[settings]\n" + "no_sql_validation = true\n" + ), + }, + expected_compiles_again=True, + ), + WorkerExecutionTestCase( + "a job with the compile cache off compiles again instead of reusing the verified compile", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + files={ + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n" + "[settings]\n" + "no_cache = true\n" + ), + }, + expected_compiles_again=True, ), WorkerExecutionTestCase( "a selected upstream source succeeds once the build returns", From e3542e2d1af78a1f93d9e889fb52236c48b8d459 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:26:50 +0000 Subject: [PATCH 3/3] fix(seed): drop a replaced seed table once and set the fingerprint table up once per run Each Snowflake seed load ran DROP TABLE IF EXISTS and then DROP VIEW IF EXISTS. The second can never do anything: when the first succeeds no view of that name exists, and when the destination is a view the first already fails. Seed loads now issue only the table drop, with the same error for a view destination as before. Each seed also set up the fingerprint table before writing its row. Seeds in a build or a seed run now share the run's fingerprint writes, as models do: the first writer sets the table up, concurrent writers wait for it, and later writers insert directly and set the table up only if the insert fails. Every seed still records its row. A 93-seed run issues 442 statements instead of 627. --- .../snowflake/classes/snowflake_adapter.py | 3 +- .../executor/build/classes/build_scheduler.py | 1 + .../executor/pipeline/_helpers/seeding.py | 5 ++ .../executor/seed/_helpers/fingerprinting.py | 42 +++++++++++----- src/sqlbuild/executor/seed/main/_execute.py | 3 ++ .../adapters/snowflake/_test_types.py | 6 +++ .../adapters/snowflake/test_client.py | 43 ++++++++++++++++ .../executor/pipeline/_helpers/_test_types.py | 11 +++++ .../executor/pipeline/_helpers/helpers.py | 45 +++++++++++++++++ .../pipeline/_helpers/test_seeding.py | 49 +++++++++++++++++++ 10 files changed, 193 insertions(+), 15 deletions(-) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index a0afac9244..3317a3cb44 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3434,10 +3434,9 @@ def load_seed( ) -> None: del infer_types if replace: - self.drop( + self.drop_table( connection=connection, destination=destination, - if_exists=True, statement_recorder=statement_recorder, ) column_defs: str = ", ".join(f"{col.name} {col.type}" for col in columns) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 538d3971b9..5790b24ac8 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1031,6 +1031,7 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed run_id=self._run_id, query_change_tracking=self._query_change_tracking, schema_prepared=self._schema_prepared, + fingerprint_writes=self._fingerprint_writes, ) if result.status == ExecutionStatus.SUCCESS: result = self._with_seed_audits(key=key, result=result, connection=connection) diff --git a/src/sqlbuild/executor/pipeline/_helpers/seeding.py b/src/sqlbuild/executor/pipeline/_helpers/seeding.py index b485db73b3..64f1f1a073 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/seeding.py +++ b/src/sqlbuild/executor/pipeline/_helpers/seeding.py @@ -16,6 +16,7 @@ from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( raise_warehouse_connection_failure, ) +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.main._execute import execute_seed from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback @@ -64,6 +65,7 @@ def run_seed_pipeline( indexed_entry: tuple[int, SeedPlanEntry] for indexed_entry in enumerate(seed_entries): pending.put(indexed_entry) + fingerprint_writes: FingerprintWrites = FingerprintWrites() try: with ThreadPoolExecutor(max_workers=effective_concurrency) as pool: futures: tuple[Future[tuple[tuple[int, SeedExecutionResult], ...]], ...] = tuple( @@ -73,6 +75,7 @@ def run_seed_pipeline( copy_context().run, _run_seed_worker, pending=pending, + fingerprint_writes=fingerprint_writes, adapter=adapter, connection=connections[worker_index], run_id=run_id, @@ -101,6 +104,7 @@ def run_seed_pipeline( def _run_seed_worker( *, pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]], + fingerprint_writes: FingerprintWrites, adapter: BaseAdapter, connection: Any, run_id: str, @@ -130,6 +134,7 @@ def _run_seed_worker( statement_recorder=StatementRecorder(), run_id=run_id, query_change_tracking=query_change_tracking, + fingerprint_writes=fingerprint_writes, ) if result.status == ExecutionStatus.FAILED: lifecycle.failed(error_code=result.error_code) diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index 02a44cc063..6e12e0eefe 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -3,11 +3,14 @@ from __future__ import annotations import hashlib +from contextlib import AbstractContextManager, nullcontext from datetime import UTC, datetime from typing import Any from sqlbuild._native import NODE_TYPE_SEED, Fingerprint, SeedPlanEntry, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites +from sqlbuild.executor.run.types import FingerprintTableLocation def try_write_seed_fingerprint( @@ -17,8 +20,9 @@ def try_write_seed_fingerprint( connection: Any, run_id: str, query_change_tracking: bool, + fingerprint_writes: FingerprintWrites | None = None, ) -> tuple[str, ...]: - """Write active-target seed identity after a successful load, returning warnings.""" + """Write a loaded seed's identity on the active target, setting each table up once per run.""" if not query_change_tracking: return () @@ -48,19 +52,31 @@ def try_write_seed_fingerprint( metadata_json=seed_entry.fingerprint_metadata_json, ts=datetime.now(tz=UTC), ) - _ = write_fingerprint( - connection=connection, - execute=adapter.execute, - database=seed_entry.destination.database, - schema=target_schema, - fingerprint=fingerprint, - render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - read_columns=adapter.get_columns, - render_add_columns=adapter.render_add_columns, - render_create_table_sql=adapter.render_create_fingerprint_table_sql, - render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, + location: FingerprintTableLocation = (seed_entry.destination.database, target_schema) + setting_up: AbstractContextManager[bool] = ( + nullcontext(True) + if fingerprint_writes is None + else fingerprint_writes.setting_up(location=location) ) + with setting_up as requested_setup: + set_up: bool = write_fingerprint( + ensure_table=requested_setup, + connection=connection, + execute=adapter.execute, + database=seed_entry.destination.database, + schema=target_schema, + fingerprint=fingerprint, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, + ) + if fingerprint_writes is not None: + fingerprint_writes.written( + location=location, requested_setup=requested_setup, set_up=set_up + ) except Exception as exc: return ( f"fingerprint write failed for seed '{seed_entry.name}'; " diff --git a/src/sqlbuild/executor/seed/main/_execute.py b/src/sqlbuild/executor/seed/main/_execute.py index b4ddb026cd..3d08e53e98 100644 --- a/src/sqlbuild/executor/seed/main/_execute.py +++ b/src/sqlbuild/executor/seed/main/_execute.py @@ -12,6 +12,7 @@ resolve_relation_location_qualified_name, ) from sqlbuild.executor.build.models import SeedExecutionResult +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed._helpers.fingerprinting import try_write_seed_fingerprint from sqlbuild.executor.seed.constants import SEED_LOAD_FAILED_CODE @@ -26,6 +27,7 @@ def execute_seed( run_id: str = "", query_change_tracking: bool = False, schema_prepared: bool = False, + fingerprint_writes: FingerprintWrites | None = None, ) -> SeedExecutionResult: """Load one seed into the warehouse.""" @@ -56,6 +58,7 @@ def execute_seed( connection=connection, run_id=run_id, query_change_tracking=query_change_tracking, + fingerprint_writes=fingerprint_writes, ) except Exception as exc: return SeedExecutionResult( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index a336d3e62d..f675a0fd21 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -142,6 +142,12 @@ class SnowflakeLoadSeedTestCase: expected_rows: list[tuple[object, ...]] +@dataclass(frozen=True) +class SnowflakeSeedReplaceTestCase: + description: str + expected_statements: tuple[str, ...] + + @dataclass(frozen=True) class SnowflakeTableFreshnessMetadataTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 5c64c55bcf..0f5530e02f 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -48,6 +48,7 @@ SnowflakeRenderTableFunctionTestCase, SnowflakeSchemaDiffTestCase, SnowflakeSchemaExistsTestCase, + SnowflakeSeedReplaceTestCase, SnowflakeTableFreshnessBatchTestCase, SnowflakeTableFreshnessMetadataErrorTestCase, SnowflakeTableFreshnessMetadataTestCase, @@ -1408,3 +1409,45 @@ def test_given_schema_when_checking_existence_then_show_needs_no_running_warehou assert exists is test_case.expected_exists assert test_case.expected_sql_fragment in str(cast(Any, test_case.cursor).executed_sql) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeSeedReplaceTestCase( + description="one table drop clears the old table, with no separate view drop", + expected_statements=( + "DROP TABLE IF EXISTS dev.waffle_types", + "CREATE TRANSIENT TABLE dev.waffle_types (id INTEGER, name VARCHAR)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_seed_replacing_its_table_when_loading_then_it_drops_only_the_table( + test_case: SnowflakeSeedReplaceTestCase, + tmp_path: Path, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + cursor: FakeSnowflakeDescribeCursor = FakeSnowflakeDescribeCursor(description=()) + connection: FakeSnowflakeDescribeConnection = FakeSnowflakeDescribeConnection(cursor) + seed_file: Path = tmp_path / "waffle_types.csv" + seed_file.write_text("id,name\n1,Liege\n", encoding="utf-8") + executed: list[str] = [] + + with patch.object( + adapter, "execute", side_effect=lambda *, connection, sql: executed.append(sql) + ): + adapter.load_seed( + connection=connection, + destination="dev.waffle_types", + file_path=seed_file, + columns=( + ColumnInfo(name="id", type="INTEGER"), + ColumnInfo(name="name", type="VARCHAR"), + ), + replace=True, + statement_recorder=StatementRecorder(), + ) + + assert tuple(executed) == test_case.expected_statements, test_case.description diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py index 69b846bda2..f76e88ee7f 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py @@ -150,6 +150,17 @@ class SeedPipelineQueueTestCase: expected_load_order: tuple[str, ...] +@dataclass(frozen=True) +class SeedFingerprintSetupTestCase: + """Concurrent seeds on an empty schema: how often the fingerprint table is set up.""" + + description: str + seed_names: tuple[str, ...] + max_concurrency: int + expected_table_setups: int + expected_fingerprint_rows: int + + @dataclass(frozen=True) class SeedPipelineConcurrencyTestCase: """One seed pipeline concurrency orchestration case.""" diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 16ff0549c2..e1574ef41d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -35,6 +35,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -455,3 +456,47 @@ def describe_lifecycle_events(events: list[LifecycleEvent]) -> list[str]: f"kind={event.payload.get('statement_kind')} resource={event.resource_id}" for event in events ] + + +class RecordingDuckDbAdapter(DuckDbAdapter): + """A real DuckDB adapter that records every statement it executes.""" + + def __init__(self) -> None: + super().__init__() + self.statements: list[str] = [] + self._statements_lock: threading.Lock = threading.Lock() + + def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any: + with self._statements_lock: + self.statements.append(sql) + return super().execute(connection=connection, sql=sql, **kwargs) + + +def build_tracked_seed_plan(*, seed_names: tuple[str, ...], seed_dir: Path) -> PlanOutput: + """Build a seed plan on schema main whose seeds carry identity, with their CSV files.""" + + _ = [ + (seed_dir / f"{name}.csv").write_text("id,name\n1,Liege\n", encoding="utf-8") + for name in seed_names + ] + return PlanOutput( + seed_entries=tuple( + SeedPlanEntry( + key=CompiledObjectKey(resource_type=CompiledResourceType.SEED, name=name), + name=name, + destination=CompiledRelationLocation( + database=None, schema="main", name=name, qualified_name=f"main.{name}" + ), + file_path=seed_dir / f"{name}.csv", + columns=( + ColumnInfo(name="id", type="INTEGER"), + ColumnInfo(name="name", type="VARCHAR"), + ), + csv_settings=DEFAULT_SEED_CSV_SETTINGS, + fingerprint_definition=f"seed {name}", + fingerprint_version_hash=f"version-{name}", + fingerprint_metadata_json="{}", + ) + for name in seed_names + ), + ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py index c1ba05633d..b867d421cd 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from pathlib import Path from typing import Any, cast from unittest.mock import patch @@ -18,14 +19,17 @@ invocation_scope, ) from tests.unit.src.sqlbuild.executor.pipeline._helpers._test_types import ( + SeedFingerprintSetupTestCase, SeedPipelineConcurrencyTestCase, SeedPipelineLifecycleTestCase, SeedPipelineQueueTestCase, ) from tests.unit.src.sqlbuild.executor.pipeline._helpers.helpers import ( + RecordingDuckDbAdapter, SeedPipelineTestAdapter, SlowSeedPipelineAdapter, build_seed_plan, + build_tracked_seed_plan, ) @@ -199,3 +203,48 @@ def test_given_a_slow_seed_when_running_seed_pipeline_then_other_seeds_take_the_ assert tuple(adapter.order) == test_case.expected_load_order, test_case.description assert tuple(result.seed_name for result in results) == test_case.seed_names + + +@pytest.mark.parametrize( + "test_case", + [ + SeedFingerprintSetupTestCase( + description="concurrent seeds on an empty schema set the table up once and each " + "record a row", + seed_names=("seed_a", "seed_b", "seed_c", "seed_d"), + max_concurrency=2, + expected_table_setups=1, + expected_fingerprint_rows=4, + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_seeds_when_recording_identity_then_the_fingerprint_table_is_set_up_once( + test_case: SeedFingerprintSetupTestCase, + tmp_path: Path, +) -> None: + adapter: RecordingDuckDbAdapter = RecordingDuckDbAdapter() + config: dict[str, object] = {"database": str(tmp_path / "warehouse.duckdb")} + + results: tuple[SeedExecutionResult, ...] = run_seed_pipeline( + plan=build_tracked_seed_plan(seed_names=test_case.seed_names, seed_dir=tmp_path), + connection_config=config, + adapter=adapter, + max_concurrency=test_case.max_concurrency, + run_id="seed-fingerprint-run", + query_change_tracking=True, + ) + + assert {result.status.value for result in results} == {"success"}, results + assert [result.warning_messages for result in results] == [()] * len(test_case.seed_names) + setups: int = sum( + sql.startswith("CREATE TABLE IF NOT EXISTS main._sqlbuild_fingerprints") + for sql in adapter.statements + ) + assert setups == test_case.expected_table_setups, adapter.statements + connection: Any = adapter.connect(config) + rows: list[tuple[Any, ...]] = adapter.execute( + connection=connection, sql="SELECT count(*) FROM main._sqlbuild_fingerprints" + ).fetchall() + adapter.close(connection) + assert rows == [(test_case.expected_fingerprint_rows,)]