Skip to content

Commit ea02cab

Browse files
committed
fix(build): fail a seed whose error-severity audit fails
The seed was reported as a success before its audits ran, so server runs showed it succeeded while its dependants were skipped. The audits now run first and a failing one turns the seed result into a failure with R006, as a failing final audit does for a model.
1 parent e7a095a commit ea02cab

8 files changed

Lines changed: 168 additions & 16 deletions

File tree

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
seeds:
2+
- name: order_codes
3+
description: Order codes, one of which the audit does not accept.
4+
columns:
5+
- name: code
6+
type: VARCHAR
7+
audits:
8+
- accepted_values:
9+
values: [A, B]

‎src/sqlbuild/executor/build/classes/build_scheduler.py‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder
105105
from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites
106106
from sqlbuild.executor.run.classes.runtime_watermark_resolver import RuntimeWatermarkResolver
107+
from sqlbuild.executor.run.constants import RUN_AUDIT_FAILED_CODE
107108
from sqlbuild.executor.run.models import (
108109
BatchWindow,
109110
BoundViewGuard,
@@ -258,6 +259,7 @@ def __init__(
258259
self._load_results: list[LoadExecutionResult] = list(initial_state.initial_load_results)
259260
self._test_results: list[SqlTestExecutionResult] = []
260261
self._source_audit_results: list[AuditExecutionResult] = []
262+
self._seed_audit_results: dict[CompiledObjectKey, tuple[AuditExecutionResult, ...]] = {}
261263

262264
self._executed_source_audits: set[str] = set()
263265
self._failed_sources: set[str] = set()
@@ -1004,6 +1006,8 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed
10041006
query_change_tracking=self._query_change_tracking,
10051007
schema_prepared=self._schema_prepared,
10061008
)
1009+
if result.status == ExecutionStatus.SUCCESS:
1010+
result = self._with_seed_audits(key=key, result=result, connection=connection)
10071011
duration: int = int((time.monotonic() - start) * 1000)
10081012
completed_result: SeedExecutionResult = dataclasses.replace(result, duration_ms=duration)
10091013
log_debug_event(
@@ -1346,12 +1350,11 @@ def _handle_completion(
13461350
failed: bool = False
13471351

13481352
if isinstance(result, SeedExecutionResult):
1353+
self._source_audit_results.extend(self._seed_audit_results.pop(key, ()))
13491354
self._seed_results.append(result)
13501355
if self._on_node_complete is not None:
13511356
self._on_node_complete(result)
13521357
failed = result.status == ExecutionStatus.FAILED
1353-
if result.status == ExecutionStatus.SUCCESS and self._run_seed_audits(key=key):
1354-
failed = True
13551358

13561359
elif isinstance(result, SqlTestExecutionResult):
13571360
self._test_results.append(result)
@@ -1404,26 +1407,38 @@ def _handle_completion(
14041407
self._stop = True
14051408
self._mark_complete(key)
14061409

1407-
def _run_seed_audits(self, *, key: CompiledObjectKey) -> bool:
1408-
"""Run a loaded seed's audits and return whether an error-severity audit failed."""
1410+
def _with_seed_audits(
1411+
self, *, key: CompiledObjectKey, result: SeedExecutionResult, connection: Any
1412+
) -> SeedExecutionResult:
1413+
"""Run a loaded seed's audits within its attempt; an error-severity failure fails it."""
14091414

14101415
if not self._run_audits:
1411-
return False
1416+
return result
14121417
audits: tuple[AuditPlanEntry, ...] = self._indexes.seed_audits_by_seed.get(key.name, ())
14131418
if not audits:
1414-
return False
1419+
return result
14151420
results: tuple[AuditExecutionResult, ...] = run_end_audits(
14161421
end_audits=audits,
14171422
adapter=self._adapter,
1418-
connection=self._scheduler_connection,
1423+
connection=connection,
14191424
model_locations=self._plan.model_locations,
14201425
seed_locations=self._plan.seed_locations,
14211426
source_map=self._plan.source_map,
14221427
run_id=self._run_id,
14231428
quality_scope="seed",
14241429
)
1425-
self._source_audit_results.extend(results)
1426-
return any(result.outcome == AuditOutcome.ERROR for result in results)
1430+
self._seed_audit_results[key] = results
1431+
if not any(audit.outcome == AuditOutcome.ERROR for audit in results):
1432+
return result
1433+
return dataclasses.replace(
1434+
result,
1435+
status=ExecutionStatus.FAILED,
1436+
error_code=RUN_AUDIT_FAILED_CODE,
1437+
error_message=(
1438+
f"audit for seed '{result.seed_name}' failed after loading "
1439+
"with severity level: error"
1440+
),
1441+
)
14271442

14281443
def _mark_complete(self, key: CompiledObjectKey) -> None:
14291444
"""Mark a key complete and enqueue any downstream that become ready."""

‎tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,6 +1042,14 @@ class AttachedAuditGateSummaryE2ETestCase:
10421042
expected_summary: str
10431043

10441044

1045+
@dataclass(frozen=True)
1046+
class AttachedAuditGateAttemptE2ETestCase:
1047+
description: str
1048+
target_kind: str
1049+
order_code: str
1050+
expected_target_event: tuple[str, str | None]
1051+
1052+
10451053
@dataclass(frozen=True)
10461054
class AttachedAuditGateCycleE2ETestCase:
10471055
description: str

‎tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -639,6 +639,15 @@ def build_check_outcomes(stdout: str) -> dict[tuple[object, object], tuple[objec
639639
}
640640

641641

642+
def integration_attempt_outcomes(records_path: Path) -> dict[object, tuple[object, object]]:
643+
"""Map each resource ID to its attempt's event type and error code from integration records."""
644+
645+
return {
646+
record["resource_id"]: (record["event_type"], record.get("error_code"))
647+
for record in map(json.loads, records_path.read_text(encoding="utf-8").splitlines())
648+
}
649+
650+
642651
def build_asset_names(stdout: str) -> tuple[str, ...]:
643652
"""Return the asset names reported by build JSON output."""
644653

‎tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import pytest
99

1010
from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
11+
AttachedAuditGateAttemptE2ETestCase,
1112
AttachedAuditGateBuildE2ETestCase,
1213
AttachedAuditGateCycleE2ETestCase,
1314
AttachedAuditGateNoAuditsE2ETestCase,
@@ -20,6 +21,7 @@
2021
ATTACHED_AUDIT_GATE_DATABASE,
2122
build_asset_names,
2223
build_check_outcomes,
24+
integration_attempt_outcomes,
2325
prepare_attached_audit_gate_project,
2426
prepare_audit_read_plan_project,
2527
prepare_nested_source_gate_project,
@@ -182,11 +184,11 @@ def test_given_attached_audit_reading_another_resource_when_building_then_it_gat
182184
expected_summary="PASS=1 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=3",
183185
),
184186
AttachedAuditGateSummaryE2ETestCase(
185-
description="failing seed audit counts as a failure",
187+
description="failing seed audit fails its seed",
186188
target_kind="seed",
187189
order_code="Z",
188190
expected_exit_code=1,
189-
expected_summary="PASS=2 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=4",
191+
expected_summary="PASS=1 WARN=0 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=4",
190192
),
191193
],
192194
ids=lambda case: case.description,
@@ -208,6 +210,48 @@ def test_given_attached_audit_outcome_when_building_in_terminal_then_summary_cou
208210
assert test_case.expected_summary in output
209211

210212

213+
@pytest.mark.parametrize(
214+
"test_case",
215+
[
216+
AttachedAuditGateAttemptE2ETestCase(
217+
description="passing seed audit completes the seed's attempt",
218+
target_kind="seed",
219+
order_code="A",
220+
expected_target_event=("resource_attempt_completed", None),
221+
),
222+
AttachedAuditGateAttemptE2ETestCase(
223+
description="failing seed audit fails the seed's attempt",
224+
target_kind="seed",
225+
order_code="Z",
226+
expected_target_event=("resource_attempt_failed", "R006"),
227+
),
228+
],
229+
ids=lambda case: case.description,
230+
)
231+
def test_given_attached_audit_outcome_when_building_then_the_target_attempt_records_it(
232+
test_case: AttachedAuditGateAttemptE2ETestCase,
233+
tmp_path: Path,
234+
) -> None:
235+
project_dir: Path = prepare_attached_audit_gate_project(
236+
tmp_path=tmp_path, target_kind=test_case.target_kind, order_code=test_case.order_code
237+
)
238+
records_path: Path = tmp_path / "integration_results.jsonl"
239+
240+
result: subprocess.CompletedProcess[str] = run_sqb(
241+
command=("--no-color", "build"),
242+
project_dir=project_dir,
243+
env={"SQLBUILD_INTEGRATION_RESULT_PATH": str(records_path)},
244+
)
245+
246+
assert (
247+
integration_attempt_outcomes(records_path).get("seed:order_codes")
248+
== test_case.expected_target_event
249+
), (
250+
test_case.description,
251+
result.stdout + result.stderr,
252+
)
253+
254+
211255
@pytest.mark.parametrize(
212256
"test_case",
213257
[

‎tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,15 @@
6161
severity="error",
6262
concurrency=1,
6363
expected_attachment_kind="seed",
64-
expected_asset_statuses={"order_codes": "success", "coded_orders": "skipped"},
64+
expected_asset_statuses={"order_codes": "failed", "coded_orders": "skipped"},
6565
),
6666
AuditExecutionErrorE2ETestCase(
6767
description="seed audit with error severity at concurrency 4",
6868
audit_kind="seed",
6969
severity="error",
7070
concurrency=4,
7171
expected_attachment_kind="seed",
72-
expected_asset_statuses={"order_codes": "success", "coded_orders": "skipped"},
72+
expected_asset_statuses={"order_codes": "failed", "coded_orders": "skipped"},
7373
),
7474
AuditExecutionErrorE2ETestCase(
7575
description="end audit with error severity at concurrency 1",
@@ -133,13 +133,14 @@ def test_given_audit_query_raises_when_building_then_it_reports_a_failed_audit(
133133
),
134134
),
135135
AuditExecutionErrorTerminalE2ETestCase(
136-
description="seed audit error is listed and counted as a failure",
136+
description="seed audit error is listed and fails its seed",
137137
audit_kind="seed",
138138
expected_fragments=(
139139
"Failures:",
140140
"broken_check on order_codes (audit)",
141+
"order_codes (seed)",
141142
"missing_lookup",
142-
"PASS=1 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=3",
143+
"PASS=0 WARN=0 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=3",
143144
),
144145
),
145146
AuditExecutionErrorTerminalE2ETestCase(

‎tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,11 @@
4949
audit_sql=_RELATION_AUDIT_SQL,
5050
seed_csv="waffle_type_id,price_cents\n1,450\n2,-1\n",
5151
expected_exit_code=1,
52-
expected_output_fragments=("expression_is_true (waffle_types)", "FAIL=1"),
52+
expected_output_fragments=(
53+
"expression_is_true (waffle_types)",
54+
"waffle_types (seed)",
55+
"FAIL=2",
56+
),
5357
),
5458
SeedYamlAuditE2ETestCase(
5559
description="model-only audit on a seed points at @relation",

‎tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,68 @@ def test_given_a_stale_source_when_a_worker_builds_then_blocked_nodes_finish_ski
263263
assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows
264264

265265

266+
@pytest.mark.parametrize(
267+
"test_case",
268+
[
269+
WorkerExecutionTestCase(
270+
description="a failing seed audit fails its seed, and what reads it is skipped",
271+
model_sql="SELECT 3 AS amount",
272+
warehouse_query="SELECT count(*) FROM duckdb_tables() WHERE table_name='coded_orders'",
273+
expected_status="partial",
274+
expected_rows=[(0,)],
275+
files={
276+
"seeds/order_codes.csv": "code\nA\nZ\n",
277+
"seeds/order_codes.yml": Path(
278+
"benchmarks/duckshop/edge_cases/orchestration/worker_failed_seed_audit.yml"
279+
).read_text(),
280+
"models/coded_orders.sql": (
281+
"MODEL (description 'Coded orders.', materialized table);\n"
282+
'SELECT code FROM __seed("order_codes")\n'
283+
),
284+
"orchestration/orders.py": (
285+
"from sqlbuild.orchestration import job, schedule\n"
286+
'annual_orders = job("annual_orders", '
287+
'select="orders order_codes coded_orders", '
288+
'description="Build orders.", '
289+
'schedule=schedule(cron="0 0 * * *", timezone="UTC", '
290+
'starts_at="2000-01-01T00:00:00Z"))\n'
291+
),
292+
},
293+
)
294+
],
295+
ids=lambda case: case.description,
296+
)
297+
def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_with_the_audit_error(
298+
test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
299+
) -> None:
300+
with scheduled_worker(tmp_path, monkeypatch, test_case) as worker:
301+
run: dict[str, Any] = wait_for_run(
302+
worker.api, worker.process, worker.log, job_key="annual_orders"
303+
)
304+
assert run["status"] == test_case.expected_status, worker.log.read_text()
305+
with sqlite3.connect(worker.control_store) as control:
306+
nodes: list[tuple[str, str, str | None, str | None]] = control.execute(
307+
"SELECT n.node_key, n.status, n.error_code, "
308+
"json_extract(e.payload, '$.change.change.error_message') "
309+
"FROM sqb_run_nodes n LEFT JOIN recall_events e "
310+
"ON e.storage_id=n.finished_storage_id "
311+
"WHERE n.run_id=? AND n.kind='node' ORDER BY n.node_key",
312+
(run["run_id"],),
313+
).fetchall()
314+
assert nodes == [
315+
("model:coded_orders", "skipped", None, None),
316+
("model:orders", "success", None, None),
317+
(
318+
"seed:order_codes",
319+
"failed",
320+
"R006",
321+
"audit for seed 'order_codes' failed after loading with severity level: error",
322+
),
323+
], worker.log.read_text()
324+
with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse:
325+
assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows
326+
327+
266328
@pytest.mark.parametrize(
267329
"test_case",
268330
[

0 commit comments

Comments
 (0)