Skip to content

Commit e3542e2

Browse files
committed
fix(seed): drop a replaced seed table once and set the fingerprint table up once per run
Each Snowflake seed load ran DROP TABLE IF EXISTS and then DROP VIEW IF EXISTS. The second can never do anything: when the first succeeds no view of that name exists, and when the destination is a view the first already fails. Seed loads now issue only the table drop, with the same error for a view destination as before. Each seed also set up the fingerprint table before writing its row. Seeds in a build or a seed run now share the run's fingerprint writes, as models do: the first writer sets the table up, concurrent writers wait for it, and later writers insert directly and set the table up only if the insert fails. Every seed still records its row. A 93-seed run issues 442 statements instead of 627.
1 parent f9f7bab commit e3542e2

10 files changed

Lines changed: 193 additions & 15 deletions

File tree

‎src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3434,10 +3434,9 @@ def load_seed(
34343434
) -> None:
34353435
del infer_types
34363436
if replace:
3437-
self.drop(
3437+
self.drop_table(
34383438
connection=connection,
34393439
destination=destination,
3440-
if_exists=True,
34413440
statement_recorder=statement_recorder,
34423441
)
34433442
column_defs: str = ", ".join(f"{col.name} {col.type}" for col in columns)

‎src/sqlbuild/executor/build/classes/build_scheduler.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1031,6 +1031,7 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed
10311031
run_id=self._run_id,
10321032
query_change_tracking=self._query_change_tracking,
10331033
schema_prepared=self._schema_prepared,
1034+
fingerprint_writes=self._fingerprint_writes,
10341035
)
10351036
if result.status == ExecutionStatus.SUCCESS:
10361037
result = self._with_seed_audits(key=key, result=result, connection=connection)

‎src/sqlbuild/executor/pipeline/_helpers/seeding.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import (
1717
raise_warehouse_connection_failure,
1818
)
19+
from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites
1920
from sqlbuild.executor.scheduling.types import ExecutionStatus
2021
from sqlbuild.executor.seed.main._execute import execute_seed
2122
from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback
@@ -64,6 +65,7 @@ def run_seed_pipeline(
6465
indexed_entry: tuple[int, SeedPlanEntry]
6566
for indexed_entry in enumerate(seed_entries):
6667
pending.put(indexed_entry)
68+
fingerprint_writes: FingerprintWrites = FingerprintWrites()
6769
try:
6870
with ThreadPoolExecutor(max_workers=effective_concurrency) as pool:
6971
futures: tuple[Future[tuple[tuple[int, SeedExecutionResult], ...]], ...] = tuple(
@@ -73,6 +75,7 @@ def run_seed_pipeline(
7375
copy_context().run,
7476
_run_seed_worker,
7577
pending=pending,
78+
fingerprint_writes=fingerprint_writes,
7679
adapter=adapter,
7780
connection=connections[worker_index],
7881
run_id=run_id,
@@ -101,6 +104,7 @@ def run_seed_pipeline(
101104
def _run_seed_worker(
102105
*,
103106
pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]],
107+
fingerprint_writes: FingerprintWrites,
104108
adapter: BaseAdapter,
105109
connection: Any,
106110
run_id: str,
@@ -130,6 +134,7 @@ def _run_seed_worker(
130134
statement_recorder=StatementRecorder(),
131135
run_id=run_id,
132136
query_change_tracking=query_change_tracking,
137+
fingerprint_writes=fingerprint_writes,
133138
)
134139
if result.status == ExecutionStatus.FAILED:
135140
lifecycle.failed(error_code=result.error_code)

‎src/sqlbuild/executor/seed/_helpers/fingerprinting.py‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@
33
from __future__ import annotations
44

55
import hashlib
6+
from contextlib import AbstractContextManager, nullcontext
67
from datetime import UTC, datetime
78
from typing import Any
89

910
from sqlbuild._native import NODE_TYPE_SEED, Fingerprint, SeedPlanEntry, write_fingerprint
1011
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
12+
from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites
13+
from sqlbuild.executor.run.types import FingerprintTableLocation
1114

1215

1316
def try_write_seed_fingerprint(
@@ -17,8 +20,9 @@ def try_write_seed_fingerprint(
1720
connection: Any,
1821
run_id: str,
1922
query_change_tracking: bool,
23+
fingerprint_writes: FingerprintWrites | None = None,
2024
) -> tuple[str, ...]:
21-
"""Write active-target seed identity after a successful load, returning warnings."""
25+
"""Write a loaded seed's identity on the active target, setting each table up once per run."""
2226

2327
if not query_change_tracking:
2428
return ()
@@ -48,19 +52,31 @@ def try_write_seed_fingerprint(
4852
metadata_json=seed_entry.fingerprint_metadata_json,
4953
ts=datetime.now(tz=UTC),
5054
)
51-
_ = write_fingerprint(
52-
connection=connection,
53-
execute=adapter.execute,
54-
database=seed_entry.destination.database,
55-
schema=target_schema,
56-
fingerprint=fingerprint,
57-
render_qualified_name=adapter.render_qualified_name,
58-
render_framework_type=adapter.render_framework_type,
59-
read_columns=adapter.get_columns,
60-
render_add_columns=adapter.render_add_columns,
61-
render_create_table_sql=adapter.render_create_fingerprint_table_sql,
62-
render_create_index_sqls=adapter.render_create_fingerprint_index_sqls,
55+
location: FingerprintTableLocation = (seed_entry.destination.database, target_schema)
56+
setting_up: AbstractContextManager[bool] = (
57+
nullcontext(True)
58+
if fingerprint_writes is None
59+
else fingerprint_writes.setting_up(location=location)
6360
)
61+
with setting_up as requested_setup:
62+
set_up: bool = write_fingerprint(
63+
ensure_table=requested_setup,
64+
connection=connection,
65+
execute=adapter.execute,
66+
database=seed_entry.destination.database,
67+
schema=target_schema,
68+
fingerprint=fingerprint,
69+
render_qualified_name=adapter.render_qualified_name,
70+
render_framework_type=adapter.render_framework_type,
71+
read_columns=adapter.get_columns,
72+
render_add_columns=adapter.render_add_columns,
73+
render_create_table_sql=adapter.render_create_fingerprint_table_sql,
74+
render_create_index_sqls=adapter.render_create_fingerprint_index_sqls,
75+
)
76+
if fingerprint_writes is not None:
77+
fingerprint_writes.written(
78+
location=location, requested_setup=requested_setup, set_up=set_up
79+
)
6480
except Exception as exc:
6581
return (
6682
f"fingerprint write failed for seed '{seed_entry.name}'; "

‎src/sqlbuild/executor/seed/main/_execute.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
resolve_relation_location_qualified_name,
1313
)
1414
from sqlbuild.executor.build.models import SeedExecutionResult
15+
from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites
1516
from sqlbuild.executor.scheduling.types import ExecutionStatus
1617
from sqlbuild.executor.seed._helpers.fingerprinting import try_write_seed_fingerprint
1718
from sqlbuild.executor.seed.constants import SEED_LOAD_FAILED_CODE
@@ -26,6 +27,7 @@ def execute_seed(
2627
run_id: str = "",
2728
query_change_tracking: bool = False,
2829
schema_prepared: bool = False,
30+
fingerprint_writes: FingerprintWrites | None = None,
2931
) -> SeedExecutionResult:
3032
"""Load one seed into the warehouse."""
3133

@@ -56,6 +58,7 @@ def execute_seed(
5658
connection=connection,
5759
run_id=run_id,
5860
query_change_tracking=query_change_tracking,
61+
fingerprint_writes=fingerprint_writes,
5962
)
6063
except Exception as exc:
6164
return SeedExecutionResult(

‎tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,12 @@ class SnowflakeLoadSeedTestCase:
142142
expected_rows: list[tuple[object, ...]]
143143

144144

145+
@dataclass(frozen=True)
146+
class SnowflakeSeedReplaceTestCase:
147+
description: str
148+
expected_statements: tuple[str, ...]
149+
150+
145151
@dataclass(frozen=True)
146152
class SnowflakeTableFreshnessMetadataTestCase:
147153
description: str

‎tests/unit/src/sqlbuild/adapters/snowflake/test_client.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
SnowflakeRenderTableFunctionTestCase,
4949
SnowflakeSchemaDiffTestCase,
5050
SnowflakeSchemaExistsTestCase,
51+
SnowflakeSeedReplaceTestCase,
5152
SnowflakeTableFreshnessBatchTestCase,
5253
SnowflakeTableFreshnessMetadataErrorTestCase,
5354
SnowflakeTableFreshnessMetadataTestCase,
@@ -1408,3 +1409,45 @@ def test_given_schema_when_checking_existence_then_show_needs_no_running_warehou
14081409

14091410
assert exists is test_case.expected_exists
14101411
assert test_case.expected_sql_fragment in str(cast(Any, test_case.cursor).executed_sql)
1412+
1413+
1414+
@pytest.mark.parametrize(
1415+
"test_case",
1416+
[
1417+
SnowflakeSeedReplaceTestCase(
1418+
description="one table drop clears the old table, with no separate view drop",
1419+
expected_statements=(
1420+
"DROP TABLE IF EXISTS dev.waffle_types",
1421+
"CREATE TRANSIENT TABLE dev.waffle_types (id INTEGER, name VARCHAR)",
1422+
),
1423+
),
1424+
],
1425+
ids=lambda case: case.description,
1426+
)
1427+
def test_given_a_seed_replacing_its_table_when_loading_then_it_drops_only_the_table(
1428+
test_case: SnowflakeSeedReplaceTestCase,
1429+
tmp_path: Path,
1430+
) -> None:
1431+
adapter: SnowflakeAdapter = SnowflakeAdapter()
1432+
cursor: FakeSnowflakeDescribeCursor = FakeSnowflakeDescribeCursor(description=())
1433+
connection: FakeSnowflakeDescribeConnection = FakeSnowflakeDescribeConnection(cursor)
1434+
seed_file: Path = tmp_path / "waffle_types.csv"
1435+
seed_file.write_text("id,name\n1,Liege\n", encoding="utf-8")
1436+
executed: list[str] = []
1437+
1438+
with patch.object(
1439+
adapter, "execute", side_effect=lambda *, connection, sql: executed.append(sql)
1440+
):
1441+
adapter.load_seed(
1442+
connection=connection,
1443+
destination="dev.waffle_types",
1444+
file_path=seed_file,
1445+
columns=(
1446+
ColumnInfo(name="id", type="INTEGER"),
1447+
ColumnInfo(name="name", type="VARCHAR"),
1448+
),
1449+
replace=True,
1450+
statement_recorder=StatementRecorder(),
1451+
)
1452+
1453+
assert tuple(executed) == test_case.expected_statements, test_case.description

‎tests/unit/src/sqlbuild/executor/pipeline/_helpers/_test_types.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,17 @@ class SeedPipelineQueueTestCase:
150150
expected_load_order: tuple[str, ...]
151151

152152

153+
@dataclass(frozen=True)
154+
class SeedFingerprintSetupTestCase:
155+
"""Concurrent seeds on an empty schema: how often the fingerprint table is set up."""
156+
157+
description: str
158+
seed_names: tuple[str, ...]
159+
max_concurrency: int
160+
expected_table_setups: int
161+
expected_fingerprint_rows: int
162+
163+
153164
@dataclass(frozen=True)
154165
class SeedPipelineConcurrencyTestCase:
155166
"""One seed pipeline concurrency orchestration case."""

‎tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
3636
from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder
3737
from sqlbuild.adapter.contract.models import ColumnInfo
38+
from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter
3839
from sqlbuild.compiler.auditing.types import (
3940
AuditAttachmentKind,
4041
AuditOutcome,
@@ -455,3 +456,47 @@ def describe_lifecycle_events(events: list[LifecycleEvent]) -> list[str]:
455456
f"kind={event.payload.get('statement_kind')} resource={event.resource_id}"
456457
for event in events
457458
]
459+
460+
461+
class RecordingDuckDbAdapter(DuckDbAdapter):
462+
"""A real DuckDB adapter that records every statement it executes."""
463+
464+
def __init__(self) -> None:
465+
super().__init__()
466+
self.statements: list[str] = []
467+
self._statements_lock: threading.Lock = threading.Lock()
468+
469+
def execute(self, *, connection: Any, sql: str, **kwargs: Any) -> Any:
470+
with self._statements_lock:
471+
self.statements.append(sql)
472+
return super().execute(connection=connection, sql=sql, **kwargs)
473+
474+
475+
def build_tracked_seed_plan(*, seed_names: tuple[str, ...], seed_dir: Path) -> PlanOutput:
476+
"""Build a seed plan on schema main whose seeds carry identity, with their CSV files."""
477+
478+
_ = [
479+
(seed_dir / f"{name}.csv").write_text("id,name\n1,Liege\n", encoding="utf-8")
480+
for name in seed_names
481+
]
482+
return PlanOutput(
483+
seed_entries=tuple(
484+
SeedPlanEntry(
485+
key=CompiledObjectKey(resource_type=CompiledResourceType.SEED, name=name),
486+
name=name,
487+
destination=CompiledRelationLocation(
488+
database=None, schema="main", name=name, qualified_name=f"main.{name}"
489+
),
490+
file_path=seed_dir / f"{name}.csv",
491+
columns=(
492+
ColumnInfo(name="id", type="INTEGER"),
493+
ColumnInfo(name="name", type="VARCHAR"),
494+
),
495+
csv_settings=DEFAULT_SEED_CSV_SETTINGS,
496+
fingerprint_definition=f"seed {name}",
497+
fingerprint_version_hash=f"version-{name}",
498+
fingerprint_metadata_json="{}",
499+
)
500+
for name in seed_names
501+
),
502+
)

‎tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from __future__ import annotations
44

55
import json
6+
from pathlib import Path
67
from typing import Any, cast
78
from unittest.mock import patch
89

@@ -18,14 +19,17 @@
1819
invocation_scope,
1920
)
2021
from tests.unit.src.sqlbuild.executor.pipeline._helpers._test_types import (
22+
SeedFingerprintSetupTestCase,
2123
SeedPipelineConcurrencyTestCase,
2224
SeedPipelineLifecycleTestCase,
2325
SeedPipelineQueueTestCase,
2426
)
2527
from tests.unit.src.sqlbuild.executor.pipeline._helpers.helpers import (
28+
RecordingDuckDbAdapter,
2629
SeedPipelineTestAdapter,
2730
SlowSeedPipelineAdapter,
2831
build_seed_plan,
32+
build_tracked_seed_plan,
2933
)
3034

3135

@@ -199,3 +203,48 @@ def test_given_a_slow_seed_when_running_seed_pipeline_then_other_seeds_take_the_
199203

200204
assert tuple(adapter.order) == test_case.expected_load_order, test_case.description
201205
assert tuple(result.seed_name for result in results) == test_case.seed_names
206+
207+
208+
@pytest.mark.parametrize(
209+
"test_case",
210+
[
211+
SeedFingerprintSetupTestCase(
212+
description="concurrent seeds on an empty schema set the table up once and each "
213+
"record a row",
214+
seed_names=("seed_a", "seed_b", "seed_c", "seed_d"),
215+
max_concurrency=2,
216+
expected_table_setups=1,
217+
expected_fingerprint_rows=4,
218+
),
219+
],
220+
ids=lambda case: case.description,
221+
)
222+
def test_given_concurrent_seeds_when_recording_identity_then_the_fingerprint_table_is_set_up_once(
223+
test_case: SeedFingerprintSetupTestCase,
224+
tmp_path: Path,
225+
) -> None:
226+
adapter: RecordingDuckDbAdapter = RecordingDuckDbAdapter()
227+
config: dict[str, object] = {"database": str(tmp_path / "warehouse.duckdb")}
228+
229+
results: tuple[SeedExecutionResult, ...] = run_seed_pipeline(
230+
plan=build_tracked_seed_plan(seed_names=test_case.seed_names, seed_dir=tmp_path),
231+
connection_config=config,
232+
adapter=adapter,
233+
max_concurrency=test_case.max_concurrency,
234+
run_id="seed-fingerprint-run",
235+
query_change_tracking=True,
236+
)
237+
238+
assert {result.status.value for result in results} == {"success"}, results
239+
assert [result.warning_messages for result in results] == [()] * len(test_case.seed_names)
240+
setups: int = sum(
241+
sql.startswith("CREATE TABLE IF NOT EXISTS main._sqlbuild_fingerprints")
242+
for sql in adapter.statements
243+
)
244+
assert setups == test_case.expected_table_setups, adapter.statements
245+
connection: Any = adapter.connect(config)
246+
rows: list[tuple[Any, ...]] = adapter.execute(
247+
connection=connection, sql="SELECT count(*) FROM main._sqlbuild_fingerprints"
248+
).fetchall()
249+
adapter.close(connection)
250+
assert rows == [(test_case.expected_fingerprint_rows,)]

0 commit comments

Comments
 (0)