Skip to content

Commit e89dafd

Browse files
authored
test(compiler): cover every discovery input in the compiler differential (#994)
* test(compiler): group differential harness helpers into buckets Split the flat _helpers package into comparing, corpus and running buckets so it can grow without exceeding the helpers container limit. Tests mirror the new layout with local test-case types. No behaviour change. * test(compiler): cover every discovery input kind in generated seeds Add stride-forced feature blocks to the seeded project generator so any four consecutive seeds exercise incremental strategies, microbatch, snapshots, Python hooks, loaders, assets, checks, audit factories, providers, sinks, custom materializations, a project-local adapter, model schemas, list constants, cross-file macro imports, macro/udf/parameterized tests, generic audits, functions, local config, target overrides, CRLF/tab/BOM input and non-ASCII comments and strings. Seed 1 of every twelve uses a dbt ref with a synthetic manifest: compile fails with C214 by design and plan must succeed. Classify each seed's DiscoveredProjectInputs stage capture into input kinds and report coverage; --require-discovery-coverage fails the run when a kind derived from the discovery dataclass or the detail list is missing. * test(compiler): add a discovery-time failure corpus Add minimal failing projects for every discovery error code (D001-D009, D011-D016) across config, SQL, YAML, conflict and Python families, including BOM-prefixed TOML and SQL. Tighten the corpus gate: the expected code must be the first error the engine reports, and a case that intends a warning declares it separately. built-in-audit-shadow now expects S010 as the error and P003 as a warning. Add a helper that runs the corpus and returns the codes each case emits. * test(compiler): prove discovery stage captures and corpus codes Check that a project with every feature block yields a discovery capture with every collection populated, no opaque or cyclic values and callables replaced by qualified names, and that a perturbed native discovery result is reported at the discovered_project_inputs capture. Run the failure corpus for real and assert each case emits its declared first error, walking every DiscoveryError subclass to require each code. Prove a code reported only as a warning fails the gate and that missing required coverage exits 1 without reporting success. * ci: require discovery coverage in the per-PR compiler differential
1 parent 6794064 commit e89dafd

49 files changed

Lines changed: 2763 additions & 205 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/verify.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,9 @@ jobs:
291291
- name: Install dependencies
292292
run: uv sync --locked --all-groups --all-extras
293293
- name: Compare fixtures, examples, generated seeds and the failure corpus
294-
run: make compiler-differential COMPILER_DIFFERENTIAL_ARGS=--stage-captures
294+
run: >-
295+
make compiler-differential
296+
COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage"
295297
296298
format_performance:
297299
name: Formatter performance guards
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Compare what two engines produced and locate the first differences."""

scripts/compiler_differential/_helpers/compare.py renamed to scripts/compiler_differential/_helpers/comparing/compare.py

File renamed without changes.

scripts/compiler_differential/_helpers/comparison.py renamed to scripts/compiler_differential/_helpers/comparing/comparison.py

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,19 @@
55
import json
66
from collections.abc import Callable
77

8-
from scripts.compiler_differential._helpers.compare import (
8+
from scripts.compiler_differential._helpers.comparing.compare import (
99
as_json_object,
1010
first_document_difference,
1111
first_json_difference,
1212
first_text_difference,
1313
)
14-
from scripts.compiler_differential._helpers.normalize import (
14+
from scripts.compiler_differential._helpers.comparing.normalize import (
1515
normalize_artifact_text,
1616
normalize_stderr,
1717
strip_manifest_metadata,
1818
strip_report_fields,
1919
)
2020
from scripts.compiler_differential.constants import (
21-
ERROR_SEVERITY,
2221
MISSING_VALUE,
2322
MODEL_RESOURCE_TYPE,
2423
PLAN_LABEL,
@@ -70,17 +69,15 @@ def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> l
7069
]
7170

7271

73-
def diagnostic_codes(*, outcome: CommandOutcome, errors_only: bool) -> tuple[str, ...]:
74-
"""Return the diagnostic codes of a JSON compile report, in report order."""
72+
def diagnostic_codes(*, outcome: CommandOutcome, severity: str | None) -> tuple[str, ...]:
73+
"""Return a JSON compile report's diagnostic codes in order, optionally of one severity."""
7574

7675
payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {}
7776
diagnostics: object = payload.get("diagnostics")
7877
found: list[str] = []
7978
for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else ():
8079
diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic)
81-
if diagnostic is not None and (
82-
not errors_only or diagnostic.get("severity") == ERROR_SEVERITY
83-
):
80+
if diagnostic is not None and (severity is None or diagnostic.get("severity") == severity):
8481
found.append(str(diagnostic.get("code")))
8582
return tuple(found)
8683

scripts/compiler_differential/_helpers/normalize.py renamed to scripts/compiler_differential/_helpers/comparing/normalize.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
from __future__ import annotations
44

5-
from scripts.compiler_differential._helpers.compare import as_json_object
5+
from scripts.compiler_differential._helpers.comparing.compare import as_json_object
66
from scripts.compiler_differential.constants import (
77
ELAPSED_TIME_MASK,
88
ELAPSED_TIME_PATTERN,

scripts/compiler_differential/_helpers/project_comparison.py renamed to scripts/compiler_differential/_helpers/comparing/project_comparison.py

Lines changed: 52 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,13 @@
66
import time
77
from pathlib import Path
88

9-
from scripts.compiler_differential._helpers.comparison import compare_engine_runs, diagnostic_codes
10-
from scripts.compiler_differential._helpers.execution import run_engine
9+
from scripts.compiler_differential._helpers.comparing.comparison import (
10+
compare_engine_runs,
11+
diagnostic_codes,
12+
)
13+
from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
14+
from scripts.compiler_differential._helpers.running.execution import run_engine
15+
from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY
1116
from scripts.compiler_differential.models import (
1217
CommandOutcome,
1318
CorpusProject,
@@ -54,6 +59,11 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
5459
project=project.name,
5560
differences=tuple(differences),
5661
seconds=time.monotonic() - started,
62+
discovered_kinds=(
63+
project_discovery_kinds(captures=left.captures, source_dir=source_dir)
64+
if project.discovery_coverage and options.stage_captures
65+
else frozenset()
66+
),
5767
)
5868

5969

@@ -68,23 +78,21 @@ def _materialized_source(*, project: CorpusProject, case_dir: Path) -> Path:
6878

6979
def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Difference]:
7080
first: CommandOutcome = run.outcomes[0]
71-
errors: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=True)
72-
expected_code: str | None = project.expected.error_code
73-
if expected_code is not None:
74-
codes: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=False)
75-
met: bool = first.exit_code != 0 and bool(errors) and expected_code in codes
76-
return (
77-
[]
78-
if met
79-
else [
81+
errors: tuple[str, ...] = diagnostic_codes(outcome=first, severity=ERROR_SEVERITY)
82+
if project.expected.error_code is not None:
83+
return [
84+
*_failure_differences(project=project, outcome=first, errors=errors),
85+
*(
8086
_expectation_difference(
8187
project=project,
82-
location="expected failure",
83-
expected=f"non-zero exit reporting {expected_code}",
84-
actual=f"exit {first.exit_code}: {', '.join(codes) or '<no codes>'}",
88+
location=f"`{outcome.label}` expected success",
89+
expected="exit 0",
90+
actual=f"exit {outcome.exit_code}",
8591
)
86-
]
87-
)
92+
for outcome in run.outcomes
93+
if outcome.label in project.expected.succeeding_commands and outcome.exit_code != 0
94+
),
95+
]
8896
problems: list[str] = [
8997
f"`{outcome.label}` exit {outcome.exit_code}"
9098
for outcome in run.outcomes
@@ -104,6 +112,34 @@ def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Diffe
104112
]
105113

106114

115+
def _failure_differences(
116+
*, project: CorpusProject, outcome: CommandOutcome, errors: tuple[str, ...]
117+
) -> list[Difference]:
118+
expected_code: str | None = project.expected.error_code
119+
differences: list[Difference] = []
120+
if outcome.exit_code == 0 or not errors or errors[0] != expected_code:
121+
differences.append(
122+
_expectation_difference(
123+
project=project,
124+
location="expected failure",
125+
expected=f"non-zero exit whose first error is {expected_code}",
126+
actual=f"exit {outcome.exit_code}: errors {', '.join(errors) or '<none>'}",
127+
)
128+
)
129+
warning_code: str | None = project.expected.warning_code
130+
warnings: tuple[str, ...] = diagnostic_codes(outcome=outcome, severity=WARNING_SEVERITY)
131+
if warning_code is not None and warning_code not in warnings:
132+
differences.append(
133+
_expectation_difference(
134+
project=project,
135+
location="expected warning",
136+
expected=f"warning {warning_code}",
137+
actual=f"warnings {', '.join(warnings) or '<none>'}",
138+
)
139+
)
140+
return differences
141+
142+
107143
def _timeout_differences(
108144
*, project: CorpusProject, runs: tuple[EngineRun, ...]
109145
) -> list[Difference]:
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Assemble the corpus projects, including the failure corpus."""

scripts/compiler_differential/_helpers/corpus.py renamed to scripts/compiler_differential/_helpers/corpus/corpus.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import tomllib
66
from pathlib import Path
77

8-
from scripts.compiler_differential._helpers.failure_cases import all_failure_cases
8+
from scripts.compiler_differential._helpers.corpus.failure_cases import all_failure_cases
99
from scripts.compiler_differential.classes.dense_project import DenseProject
1010
from scripts.compiler_differential.classes.project_builder import ProjectBuilder
1111
from scripts.compiler_differential.constants import (
@@ -80,7 +80,9 @@ def build_corpus(
8080
CorpusProject(
8181
name=f"failure/{case.name}",
8282
commands=(COLD_COMPILE,),
83-
expected=ExpectedOutcome(error_code=case.expected_code),
83+
expected=ExpectedOutcome(
84+
error_code=case.expected_code, warning_code=case.expected_warning_code
85+
),
8486
writer=case.write,
8587
)
8688
for case in all_failure_cases()
@@ -136,6 +138,10 @@ def _seed_project(seed: int) -> CorpusProject:
136138
return CorpusProject(
137139
name=f"seed/{seed}",
138140
commands=(COLD_COMPILE, PLAN),
139-
expected=ExpectedOutcome(error_code=generated.expected_error_code),
141+
expected=ExpectedOutcome(
142+
error_code=generated.expected_error_code,
143+
succeeding_commands=generated.succeeding_commands,
144+
),
140145
writer=generated.write,
146+
discovery_coverage=True,
141147
)

0 commit comments

Comments
 (0)