diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index 305b9ce34a..f7f2d5cde5 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -273,6 +273,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("selector_files", FieldDefault::EmptyTuple), FieldSpec::new("test_timeout", FieldDefault::None), FieldSpec::new("discovered_inputs", FieldDefault::None), + FieldSpec::new("compiled_project", FieldDefault::None), ], }; const BUILD_INVOCATION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index eb8395976e..d16b491703 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -1,6 +1,10 @@ //! Compile the build plan, show it, gate it and write the compile target. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyDictMethods, PyErr, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_config::command_connection::constants::{ + CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, +}; use sqlbuild_core::build_command::constants::{ EXECUTION_LIMIT_BUILD_NOTE, MISSING_ORIGIN_BUILD_HELP, RETENTION_DECREASE_BUILD_HELP, SNAPSHOT_FULL_REFRESH_BUILD_HELP, TABLE_TYPE_DOWNGRADE_BUILD_HELP, @@ -13,7 +17,7 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::enforce_plan_policies; use crate::bindings::_helpers::build_command::policies::model_limit; use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures; -use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; +use crate::bindings::_helpers::compile_driver::plan_pipeline::planned_project; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -27,6 +31,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::manifest::main::build_manifest::build_manifest; +use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_output::build_progress::main::with_verbose_inspection::with_verbose_inspection; use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; @@ -128,22 +133,50 @@ pub(crate) fn compile_build_plan<'py>(context: &BuildContext<'py>) -> PyResult = request_options(context)?; + let held: Option> = held_compile(context, &options)?; with_verbose_inspection(py, verbose, || { - run_compile_pipeline( - py, - Some(keywords( - py, - &[ - ("discovered_inputs", context.get("discovered_inputs")?), - ("adapter", context.get("adapter")?), - ("options", request_options(context)?), - ("hooks", hooks), - ], - )?), + planned_project( + (&context.get("discovered_inputs")?, &context.get("adapter")?), + (&options, hooks), + (held.as_ref(), None), ) }) } +/// The compile a worker verified, when compiling again could not differ: no run variables, validation and cache on, no dbt. +fn held_compile<'py>( + context: &BuildContext<'py>, + options: &Bound<'py, PyAny>, +) -> PyResult>> { + let held: Bound<'py, PyAny> = native_field(&context.request, "compiled_project")?; + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + let project_config: Bound<'py, PyAny> = native_field(&discovered, "project_config")?; + let dbt_manifest: Bound<'py, PyAny> = + native_field(&native_field(&project_config, "dbt")?, "target_path")?; + if held.is_none() + || native_field(options, "cli_vars")?.is_truthy()? + || native_field(options, "no_sql_validation")?.is_truthy()? + || native_field(options, "no_cache")?.is_truthy()? + || !dbt_manifest.is_none() + { + return Ok(None); + } + let connection: Bound<'py, PyDict> = build_effective_connection_config( + &discovered, + Some(native_field(options, "selected_target")?), + None, + false, + )? + .cast_into()?; + let from_dbt_profile: bool = connection + .get_item(CONNECTION_SOURCE_KEY)? + .map(|source| source.eq(DBT_PROFILE_CONNECTION_SOURCE)) + .transpose()? + .unwrap_or(false); + Ok((!from_dbt_profile).then_some(held)) +} + /// Write the formatted plan to stdout, or stderr for debug and JSON runs. pub(crate) fn write_plan_text( context: &BuildContext<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 9985b75133..f17249275d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -345,10 +345,10 @@ pub(crate) fn run_location<'py>( arguments.value("run_id")?.extract::()?, arguments.value("claim_id")?.extract::()?, ); - let ((snapshot, blocked, _), project) = compile_location(py, &arguments, &input)?; + let ((snapshot, blocked, compiled), project) = compile_location(py, &arguments, &input)?; return execute_worker( py, - (&snapshot, project, blocked), + ((&snapshot, &compiled), project, blocked), (&arguments, &input), (&bearer, identity, recall), ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 7a88a487a1..208f2e1aee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -147,12 +147,12 @@ fn job_run_settings( Ok(settings) } -/// The frozen command's request and warehouse scope; build carries the verified inputs in its request. +/// The frozen command's request and warehouse scope; build carries the verified inputs and compiled project in its request. fn command_request<'py>( py: Python<'py>, discovered: &Bound<'py, PyAny>, arguments: &KeywordArguments<'py>, - worker: &Bound<'py, NativeOrchestrationWorker>, + (worker, compiled): (&Bound<'py, NativeOrchestrationWorker>, &Bound<'py, PyAny>), ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; @@ -230,7 +230,13 @@ fn command_request<'py>( } let mut request = Invocation::new(&args)?.value(route.arguments[0].source)?; if command == RunCommand::Build { - request = replaced(&request, &[("discovered_inputs", discovered.clone())])?; + request = replaced( + &request, + &[ + ("discovered_inputs", discovered.clone()), + ("compiled_project", compiled.clone()), + ], + )?; } Ok((request, warehouse_scope(&args)?)) } @@ -343,7 +349,11 @@ fn record_compiled_vars<'py>( pub(super) fn execute_worker<'py>( py: Python<'py>, - (snapshot, project, blocked): (&NativeOrchestrationSnapshot, ProjectAdapter<'py>, NodeKeys), + ((snapshot, compiled), project, blocked): ( + (&NativeOrchestrationSnapshot, &Bound<'py, PyAny>), + ProjectAdapter<'py>, + NodeKeys, + ), (arguments, input): (&KeywordArguments<'py>, &SnapshotInput), (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), ) -> PyResult { @@ -385,7 +395,7 @@ pub(super) fn execute_worker<'py>( return Err(PyValueError::new_err(reason)); } record_compiled_vars(py, (&owner, &frozen), (arguments, input, &project))?; - let (request, scope) = command_request(py, &discovered, arguments, &owner)?; + let (request, scope) = command_request(py, &discovered, arguments, (&owner, compiled))?; write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?; let _rules = RulesAtLoadScope::enter(); within(&scope, |_| { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index de71fe3845..60ae0b7736 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6595,6 +6595,7 @@ class BuildCommandRequest(NativeRecord): selector_files: tuple[SelectorFileSummary, ...] test_timeout: str | None discovered_inputs: DiscoveredProjectInputs | None + compiled_project: object | None def __init__( self, project_dir: Path | None = None, @@ -6637,6 +6638,7 @@ class BuildCommandRequest(NativeRecord): selector_files: tuple[SelectorFileSummary, ...] = (), test_timeout: str | None = None, discovered_inputs: DiscoveredProjectInputs | None = None, + compiled_project: object | None = None, ) -> None: ... class CloneCommandRequest(NativeRecord): diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index a0afac9244..3317a3cb44 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3434,10 +3434,9 @@ def load_seed( ) -> None: del infer_types if replace: - self.drop( + self.drop_table( connection=connection, destination=destination, - if_exists=True, statement_recorder=statement_recorder, ) column_defs: str = ", ".join(f"{col.name} {col.type}" for col in columns) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 538d3971b9..5790b24ac8 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1031,6 +1031,7 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed run_id=self._run_id, query_change_tracking=self._query_change_tracking, schema_prepared=self._schema_prepared, + fingerprint_writes=self._fingerprint_writes, ) if result.status == ExecutionStatus.SUCCESS: result = self._with_seed_audits(key=key, result=result, connection=connection) diff --git a/src/sqlbuild/executor/pipeline/_helpers/seeding.py b/src/sqlbuild/executor/pipeline/_helpers/seeding.py index 71bfa9083f..64f1f1a073 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/seeding.py +++ b/src/sqlbuild/executor/pipeline/_helpers/seeding.py @@ -2,6 +2,7 @@ from __future__ import annotations +import queue import time from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor @@ -15,6 +16,7 @@ from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( raise_warehouse_connection_failure, ) +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.main._execute import execute_seed from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback @@ -59,6 +61,11 @@ def run_seed_pipeline( if on_connection_complete is not None: on_connection_complete(effective_concurrency, elapsed_seconds=time.monotonic() - start) + pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]] = queue.SimpleQueue() + indexed_entry: tuple[int, SeedPlanEntry] + for indexed_entry in enumerate(seed_entries): + pending.put(indexed_entry) + fingerprint_writes: FingerprintWrites = FingerprintWrites() try: with ThreadPoolExecutor(max_workers=effective_concurrency) as pool: futures: tuple[Future[tuple[tuple[int, SeedExecutionResult], ...]], ...] = tuple( @@ -67,9 +74,8 @@ def run_seed_pipeline( pool.submit( copy_context().run, _run_seed_worker, - worker_index=worker_index, - effective_concurrency=effective_concurrency, - seed_entries=seed_entries, + pending=pending, + fingerprint_writes=fingerprint_writes, adapter=adapter, connection=connections[worker_index], run_id=run_id, @@ -97,21 +103,24 @@ def run_seed_pipeline( def _run_seed_worker( *, - worker_index: int, - effective_concurrency: int, - seed_entries: tuple[SeedPlanEntry, ...], + pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]], + fingerprint_writes: FingerprintWrites, adapter: BaseAdapter, connection: Any, run_id: str, query_change_tracking: bool, on_seed_complete: Callable[[SeedExecutionResult], None] | None, ) -> tuple[tuple[int, SeedExecutionResult], ...]: + """Load seeds from the shared queue until it is empty, so a slow seed holds up only itself.""" + indexed_results: list[tuple[int, SeedExecutionResult]] = [] seed_index: int entry: SeedPlanEntry - for seed_index, entry in enumerate(seed_entries): - if seed_index % effective_concurrency != worker_index: - continue + while True: + try: + seed_index, entry = pending.get_nowait() + except queue.Empty: + break with ResourceAttemptLifecycle( resource_id=f"seed:{entry.name}", resource_kind="seed", @@ -125,6 +134,7 @@ def _run_seed_worker( statement_recorder=StatementRecorder(), run_id=run_id, query_change_tracking=query_change_tracking, + fingerprint_writes=fingerprint_writes, ) if result.status == ExecutionStatus.FAILED: lifecycle.failed(error_code=result.error_code) diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index 02a44cc063..6e12e0eefe 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -3,11 +3,14 @@ from __future__ import annotations import hashlib +from contextlib import AbstractContextManager, nullcontext from datetime import UTC, datetime from typing import Any from sqlbuild._native import NODE_TYPE_SEED, Fingerprint, SeedPlanEntry, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites +from sqlbuild.executor.run.types import FingerprintTableLocation def try_write_seed_fingerprint( @@ -17,8 +20,9 @@ def try_write_seed_fingerprint( connection: Any, run_id: str, query_change_tracking: bool, + fingerprint_writes: FingerprintWrites | None = None, ) -> tuple[str, ...]: - """Write active-target seed identity after a successful load, returning warnings.""" + """Write a loaded seed's identity on the active target, setting each table up once per run.""" if not query_change_tracking: return () @@ -48,19 +52,31 @@ def try_write_seed_fingerprint( metadata_json=seed_entry.fingerprint_metadata_json, ts=datetime.now(tz=UTC), ) - _ = write_fingerprint( - connection=connection, - execute=adapter.execute, - database=seed_entry.destination.database, - schema=target_schema, - fingerprint=fingerprint, - render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - read_columns=adapter.get_columns, - render_add_columns=adapter.render_add_columns, - render_create_table_sql=adapter.render_create_fingerprint_table_sql, - render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, + location: FingerprintTableLocation = (seed_entry.destination.database, target_schema) + setting_up: AbstractContextManager[bool] = ( + nullcontext(True) + if fingerprint_writes is None + else fingerprint_writes.setting_up(location=location) ) + with setting_up as requested_setup: + set_up: bool = write_fingerprint( + ensure_table=requested_setup, + connection=connection, + execute=adapter.execute, + database=seed_entry.destination.database, + schema=target_schema, + fingerprint=fingerprint, + render_qualified_name=adapter.render_qualified_name, + render_framework_type=adapter.render_framework_type, + read_columns=adapter.get_columns, + render_add_columns=adapter.render_add_columns, + render_create_table_sql=adapter.render_create_fingerprint_table_sql, + render_create_index_sqls=adapter.render_create_fingerprint_index_sqls, + ) + if fingerprint_writes is not None: + fingerprint_writes.written( + location=location, requested_setup=requested_setup, set_up=set_up + ) except Exception as exc: return ( f"fingerprint write failed for seed '{seed_entry.name}'; " diff --git a/src/sqlbuild/executor/seed/main/_execute.py b/src/sqlbuild/executor/seed/main/_execute.py index b4ddb026cd..3d08e53e98 100644 --- a/src/sqlbuild/executor/seed/main/_execute.py +++ b/src/sqlbuild/executor/seed/main/_execute.py @@ -12,6 +12,7 @@ resolve_relation_location_qualified_name, ) from sqlbuild.executor.build.models import SeedExecutionResult +from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed._helpers.fingerprinting import try_write_seed_fingerprint from sqlbuild.executor.seed.constants import SEED_LOAD_FAILED_CODE @@ -26,6 +27,7 @@ def execute_seed( run_id: str = "", query_change_tracking: bool = False, schema_prepared: bool = False, + fingerprint_writes: FingerprintWrites | None = None, ) -> SeedExecutionResult: """Load one seed into the warehouse.""" @@ -56,6 +58,7 @@ def execute_seed( connection=connection, run_id=run_id, query_change_tracking=query_change_tracking, + fingerprint_writes=fingerprint_writes, ) except Exception as exc: return SeedExecutionResult( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index dfeb9594bc..4d306ad30b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -142,6 +142,8 @@ class WorkerExecutionTestCase: expected_run_nodes: list[tuple[str, str, str | None, str | None, str | None]] = field( default_factory=list ) + # Whether the build compiles again instead of planning from the compile the worker verified. + expected_compiles_again: bool = False @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index a2ed27e6c1..318655288f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1037,6 +1037,17 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - ).fetchall() assert contexts, log.read_text() assert all(context is not None for (context,) in contexts) + printed: str = "".join( + str(data) + for (data,) in control.execute( + "SELECT data FROM recall_output WHERE invocation_id LIKE 'worker/%' " + "ORDER BY occurred_at, sequence" + ).fetchall() + ) + assert ("Planning from the published compiled project." not in printed) == ( + test_case.expected_compiles_again + ), printed + assert ("Compiling project..." in printed) == test_case.expected_compiles_again, printed expected_run_ids: set[str] = { run_id for (run_id,) in control.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 59b3f4edd6..64da65119a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -857,6 +857,45 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "run_tests = false\n" ), }, + expected_compiles_again=True, + ), + WorkerExecutionTestCase( + "a job with SQL validation off compiles again instead of reusing the verified compile", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + files={ + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n" + "[settings]\n" + "no_sql_validation = true\n" + ), + }, + expected_compiles_again=True, + ), + WorkerExecutionTestCase( + "a job with the compile cache off compiles again instead of reusing the verified compile", + "SELECT 47 AS amount", + "SELECT amount FROM orders", + "success", + [(47,)], + files={ + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n" + "[settings]\n" + "no_cache = true\n" + ), + }, + expected_compiles_again=True, ), WorkerExecutionTestCase( "a selected upstream source succeeds once the build returns", diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index a336d3e62d..f675a0fd21 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -142,6 +142,12 @@ class SnowflakeLoadSeedTestCase: expected_rows: list[tuple[object, ...]] +@dataclass(frozen=True) +class SnowflakeSeedReplaceTestCase: + description: str + expected_statements: tuple[str, ...] + + @dataclass(frozen=True) class SnowflakeTableFreshnessMetadataTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 5c64c55bcf..0f5530e02f 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -48,6 +48,7 @@ SnowflakeRenderTableFunctionTestCase, SnowflakeSchemaDiffTestCase, SnowflakeSchemaExistsTestCase, + SnowflakeSeedReplaceTestCase, SnowflakeTableFreshnessBatchTestCase, SnowflakeTableFreshnessMetadataErrorTestCase, SnowflakeTableFreshnessMetadataTestCase, @@ -1408,3 +1409,45 @@ def test_given_schema_when_checking_existence_then_show_needs_no_running_warehou assert exists is test_case.expected_exists assert test_case.expected_sql_fragment in str(cast(Any, test_case.cursor).executed_sql) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeSeedReplaceTestCase( + description="one table drop clears the old table, with no separate view drop", + expected_statements=( + "DROP TABLE IF EXISTS dev.waffle_types", + "CREATE TRANSIENT TABLE dev.waffle_types (id INTEGER, name VARCHAR)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_seed_replacing_its_table_when_loading_then_it_drops_only_the_table( + test_case: SnowflakeSeedReplaceTestCase, + tmp_path: Path, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + cursor: FakeSnowflakeDescribeCursor = FakeSnowflakeDescribeCursor(description=()) + connection: FakeSnowflakeDescribeConnection = FakeSnowflakeDescribeConnection(cursor) + seed_file: Path = tmp_path / "waffle_types.csv" + seed_file.write_text("id,name\n1,Liege\n", encoding="utf-8") + executed: list[str] = [] + + with patch.object( + adapter, "execute", side_effect=lambda *, connection, sql: executed.append(sql) + ): + adapter.load_seed( + connection=connection, + destination="dev.waffle_types", + file_path=seed_file, + columns=( + ColumnInfo(name="id", type="INTEGER"), + ColumnInfo(name="name", type="VARCHAR"), + ), + replace=True, + statement_recorder=StatementRecorder(), + ) + + assert tuple(executed) == test_case.expected_statements, test_case.description diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py index 7853c224f1..f76e88ee7f 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py @@ -139,6 +139,28 @@ class ScenarioPlanningLifecycleTestCase: expected_terminal: str +@dataclass(frozen=True) +class SeedPipelineQueueTestCase: + """Seeds behind a slow one: which load before it finishes.""" + + description: str + seed_names: tuple[str, ...] + slow_seed: str + max_concurrency: int + expected_load_order: tuple[str, ...] + + +@dataclass(frozen=True) +class SeedFingerprintSetupTestCase: + """Concurrent seeds on an empty schema: how often the fingerprint table is set up.""" + + description: str + seed_names: tuple[str, ...] + max_concurrency: int + expected_table_setups: int + expected_fingerprint_rows: int + + @dataclass(frozen=True) class SeedPipelineConcurrencyTestCase: """One seed pipeline concurrency orchestration case.""" diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index a25935e76d..e1574ef41d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -35,6 +35,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, @@ -186,6 +187,40 @@ def load_seed( self.loads.append((destination, connection)) +def _no_signal() -> None: + """The slow seed signals nothing when it finishes.""" + + +class SlowSeedPipelineAdapter(SeedPipelineTestAdapter): + """Adapter whose slow seed finishes only after every other seed has loaded.""" + + def __init__(self, *, slow_target: str, other_count: int) -> None: + super().__init__(barrier_targets=(slow_target,)) + self.slow_target: str = slow_target + self.other_count: int = other_count + self.others_done: threading.Semaphore = threading.Semaphore(0) + self.order: list[str] = [] + + def load_seed( + self, + connection: Any, + *, + destination: str, + file_path: Path, + columns: tuple[ColumnInfo, ...], + csv_settings: SeedCsvSettings = DEFAULT_SEED_CSV_SETTINGS, + replace: bool = True, + infer_types: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + del connection, file_path, columns, csv_settings, replace, infer_types, statement_recorder + waits: int = self.other_count * (destination == self.slow_target) + released: bool = all(self.others_done.acquire(timeout=5) for _ in range(waits)) + self.order.append(destination + " timed out" * (not released)) + finish: dict[bool, Callable[[], None]] = {True: _no_signal, False: self.others_done.release} + finish[destination == self.slow_target]() + + class ScenarioPipelinePlanBuilder: """Callable plan builder that can fail for one configured scenario.""" @@ -421,3 +456,47 @@ def describe_lifecycle_events(events: list[LifecycleEvent]) -> list[str]: f"kind={event.payload.get('statement_kind')} resource={event.resource_id}" for event in events ] + + +class RecordingDuckDbAdapter(DuckDbAdapter): + """A real DuckDB adapter that records every statement it executes.""" + + def __init__(self) -> None: + super().__init__() + self.statements: list[str] = [] + self._statements_lock: threading.Lock = threading.Lock() + + def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any: + with self._statements_lock: + self.statements.append(sql) + return super().execute(connection=connection, sql=sql, **kwargs) + + +def build_tracked_seed_plan(*, seed_names: tuple[str, ...], seed_dir: Path) -> PlanOutput: + """Build a seed plan on schema main whose seeds carry identity, with their CSV files.""" + + _ = [ + (seed_dir / f"{name}.csv").write_text("id,name\n1,Liege\n", encoding="utf-8") + for name in seed_names + ] + return PlanOutput( + seed_entries=tuple( + SeedPlanEntry( + key=CompiledObjectKey(resource_type=CompiledResourceType.SEED, name=name), + name=name, + destination=CompiledRelationLocation( + database=None, schema="main", name=name, qualified_name=f"main.{name}" + ), + file_path=seed_dir / f"{name}.csv", + columns=( + ColumnInfo(name="id", type="INTEGER"), + ColumnInfo(name="name", type="VARCHAR"), + ), + csv_settings=DEFAULT_SEED_CSV_SETTINGS, + fingerprint_definition=f"seed {name}", + fingerprint_version_hash=f"version-{name}", + fingerprint_metadata_json="{}", + ) + for name in seed_names + ), + ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py index 28c6ee74a1..b867d421cd 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from pathlib import Path from typing import Any, cast from unittest.mock import patch @@ -18,12 +19,17 @@ invocation_scope, ) from tests.unit.src.sqlbuild.executor.pipeline._helpers._test_types import ( + SeedFingerprintSetupTestCase, SeedPipelineConcurrencyTestCase, SeedPipelineLifecycleTestCase, + SeedPipelineQueueTestCase, ) from tests.unit.src.sqlbuild.executor.pipeline._helpers.helpers import ( + RecordingDuckDbAdapter, SeedPipelineTestAdapter, + SlowSeedPipelineAdapter, build_seed_plan, + build_tracked_seed_plan, ) @@ -166,3 +172,79 @@ def test_given_seed_executor_raises_when_running_worker_then_attempt_fails_once_ assert tuple(event.event_type for event in resource_events) == test_case.expected_terminal_types assert {event.resource_id for event in resource_events} == test_case.expected_resource_ids assert callbacks == [] + + +@pytest.mark.parametrize( + "test_case", + [ + SeedPipelineQueueTestCase( + description="seeds behind a slow one load on the free connection meanwhile", + seed_names=("seed_slow", "seed_b", "seed_c", "seed_d"), + slow_seed="seed_slow", + max_concurrency=2, + expected_load_order=("seed_b", "seed_c", "seed_d", "seed_slow"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_slow_seed_when_running_seed_pipeline_then_other_seeds_take_the_free_connection( + test_case: SeedPipelineQueueTestCase, +) -> None: + adapter: SlowSeedPipelineAdapter = SlowSeedPipelineAdapter( + slow_target=test_case.slow_seed, other_count=len(test_case.seed_names) - 1 + ) + + results: tuple[SeedExecutionResult, ...] = run_seed_pipeline( + plan=build_seed_plan(seed_names=test_case.seed_names), + connection_config={}, + adapter=adapter, + max_concurrency=test_case.max_concurrency, + ) + + assert tuple(adapter.order) == test_case.expected_load_order, test_case.description + assert tuple(result.seed_name for result in results) == test_case.seed_names + + +@pytest.mark.parametrize( + "test_case", + [ + SeedFingerprintSetupTestCase( + description="concurrent seeds on an empty schema set the table up once and each " + "record a row", + seed_names=("seed_a", "seed_b", "seed_c", "seed_d"), + max_concurrency=2, + expected_table_setups=1, + expected_fingerprint_rows=4, + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_seeds_when_recording_identity_then_the_fingerprint_table_is_set_up_once( + test_case: SeedFingerprintSetupTestCase, + tmp_path: Path, +) -> None: + adapter: RecordingDuckDbAdapter = RecordingDuckDbAdapter() + config: dict[str, object] = {"database": str(tmp_path / "warehouse.duckdb")} + + results: tuple[SeedExecutionResult, ...] = run_seed_pipeline( + plan=build_tracked_seed_plan(seed_names=test_case.seed_names, seed_dir=tmp_path), + connection_config=config, + adapter=adapter, + max_concurrency=test_case.max_concurrency, + run_id="seed-fingerprint-run", + query_change_tracking=True, + ) + + assert {result.status.value for result in results} == {"success"}, results + assert [result.warning_messages for result in results] == [()] * len(test_case.seed_names) + setups: int = sum( + sql.startswith("CREATE TABLE IF NOT EXISTS main._sqlbuild_fingerprints") + for sql in adapter.statements + ) + assert setups == test_case.expected_table_setups, adapter.statements + connection: Any = adapter.connect(config) + rows: list[tuple[Any, ...]] = adapter.execute( + connection=connection, sql="SELECT count(*) FROM main._sqlbuild_fingerprints" + ).fetchall() + adapter.close(connection) + assert rows == [(test_case.expected_fingerprint_rows,)]