Skip to content

Commit 55f7d0f

Browse files
committed
ci: keep compiler differential failure evidence
1 parent 2cf3ecf commit 55f7d0f

9 files changed

Lines changed: 261 additions & 2 deletions

File tree

‎.github/workflows/verify.yml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,8 @@ jobs:
308308
make compiler-differential
309309
COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
310310
--require-render-coverage --require-analysis-coverage
311-
--wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json"
311+
--wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json
312+
--evidence-dir ${{ runner.temp }}/compiler-differential-evidence/preview"
312313
- name: Upload the Polyglot wheel-site and deferral report
313314
if: always()
314315
uses: actions/upload-artifact@v4
@@ -321,7 +322,16 @@ jobs:
321322
run: >-
322323
make compiler-differential-shipped
323324
COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
324-
--require-render-coverage"
325+
--require-render-coverage
326+
--evidence-dir ${{ runner.temp }}/compiler-differential-evidence/shipped"
327+
- name: Upload differing projects' targets, stderr and compiled-test stats
328+
if: failure()
329+
uses: actions/upload-artifact@v4
330+
with:
331+
name: compiler-differential-evidence-${{ github.run_id }}-${{ github.run_attempt }}
332+
path: ${{ runner.temp }}/compiler-differential-evidence
333+
if-no-files-found: warn
334+
retention-days: 30
325335

326336
compiler_differential_python_versions:
327337
name: Native compiler on Python ${{ matrix.python }}

‎scripts/compiler_differential/_helpers/comparing/project_comparison.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from scripts.compiler_differential._helpers.coverage.analysis import project_analysis_kinds
1414
from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
1515
from scripts.compiler_differential._helpers.coverage.render import project_render_kinds
16+
from scripts.compiler_differential._helpers.running.evidence import keep_failure_evidence
1617
from scripts.compiler_differential._helpers.running.execution import run_engine
1718
from scripts.compiler_differential.constants import (
1819
ERROR_SEVERITY,
@@ -65,6 +66,13 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
6566
*_corpus_differences(project=project, run=left),
6667
*compare_engine_runs(project=project.name, left=left, right=right),
6768
]
69+
if differences and options.evidence_dir is not None:
70+
_ = keep_failure_evidence(
71+
case_dir=case_dir,
72+
project_subdirectory=project.project_subdirectory,
73+
runs=(left, right),
74+
evidence_dir=options.evidence_dir,
75+
)
6876
return ProjectComparison(
6977
project=project.name,
7078
differences=tuple(differences),
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
"""Keep a differing project's engine runs where CI can upload them."""
2+
3+
from __future__ import annotations
4+
5+
import os
6+
import shutil
7+
from pathlib import Path
8+
9+
from scripts.compiler_differential.models import EngineRun
10+
11+
_TARGET_DIRECTORY: str = "target"
12+
_COMPILED_TESTS: tuple[str, ...] = ("target", "compiled", "tests")
13+
_STAT_FILE: str = "compiled-tests-stat.tsv"
14+
15+
16+
def keep_failure_evidence(
17+
*,
18+
case_dir: Path,
19+
project_subdirectory: str | None,
20+
runs: tuple[EngineRun, ...],
21+
evidence_dir: Path,
22+
) -> Path:
23+
"""Copy each engine's `target/`, every command's stderr and a compiled-test stat listing."""
24+
25+
destination: Path = evidence_dir / case_dir.name
26+
shutil.rmtree(destination, ignore_errors=True)
27+
for run in runs:
28+
project_dir: Path = case_dir / run.engine / (project_subdirectory or "")
29+
engine_dir: Path = destination / run.engine
30+
engine_dir.mkdir(parents=True)
31+
target: Path = project_dir / _TARGET_DIRECTORY
32+
if target.is_dir():
33+
_ = shutil.copytree(target, engine_dir / _TARGET_DIRECTORY, symlinks=True)
34+
for index, outcome in enumerate(run.outcomes):
35+
_ = (engine_dir / f"{index}-{outcome.label}.stderr").write_text(
36+
outcome.stderr, encoding="utf-8", errors="surrogateescape"
37+
)
38+
_ = (engine_dir / _STAT_FILE).write_text(
39+
_stat_listing(project_dir.joinpath(*_COMPILED_TESTS)), encoding="utf-8"
40+
)
41+
return destination
42+
43+
44+
def _stat_listing(root: Path) -> str:
45+
"""`relative path, size, mtime_ns, ctime_ns` for every file below `root`, sorted."""
46+
47+
rows: list[str] = ["path\tsize\tmtime_ns\tctime_ns"]
48+
for directory, _, filenames in sorted(os.walk(root)):
49+
for filename in sorted(filenames):
50+
path: Path = Path(directory, filename)
51+
status: os.stat_result = path.stat()
52+
rows.append(
53+
f"{path.relative_to(root).as_posix()}\t{status.st_size}\t"
54+
f"{status.st_mtime_ns}\t{status.st_ctime_ns}"
55+
)
56+
return "\n".join(rows) + "\n"

‎scripts/compiler_differential/_helpers/running/run.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe
3232
require_render_coverage=args.require_render_coverage,
3333
require_analysis_coverage=args.require_analysis_coverage,
3434
analysis_records=args.wheel_site_report is not None,
35+
evidence_dir=args.evidence_dir,
3536
)
3637

3738

‎scripts/compiler_differential/main/differential.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,13 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace:
187187
help="extra environment for one engine's processes",
188188
)
189189
parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here")
190+
parser.add_argument(
191+
"--evidence-dir",
192+
type=Path,
193+
default=None,
194+
metavar="PATH",
195+
help="copy each differing project's target/, stderr and compiled-test stats to PATH",
196+
)
190197
parser.add_argument("--python", type=Path, default=Path(sys.executable))
191198
args: argparse.Namespace = parser.parse_args(argv)
192199
for flag, required in (

‎scripts/compiler_differential/models.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ class DifferentialOptions:
118118
require_render_coverage: bool = False
119119
require_analysis_coverage: bool = False
120120
analysis_records: bool = False
121+
evidence_dir: Path | None = None
121122

122123

123124
@dataclass(frozen=True)

‎tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,13 @@ class AnalysisRecordsTestCase:
7575
expected_wheel_sites: dict[tuple[str, str], int]
7676
expected_deferrals: dict[tuple[str, str], int]
7777
expected_lines: tuple[str, ...]
78+
79+
80+
@dataclass(frozen=True)
81+
class FailureEvidenceTestCase:
82+
"""Per-engine warm-compile stderr and the evidence files a comparison keeps."""
83+
84+
description: str
85+
stderr_by_engine: dict[str, str]
86+
expected_files: tuple[str, ...]
87+
expected_evidence_engines: tuple[str, ...]
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
"""Fake engine runs for the differential harness running tests."""
2+
3+
from __future__ import annotations
4+
5+
from collections.abc import Callable
6+
from pathlib import Path
7+
8+
from scripts.compiler_differential.models import (
9+
CommandOutcome,
10+
CorpusProject,
11+
DifferentialOptions,
12+
EngineRun,
13+
)
14+
15+
ORDERS_TEST_SQL: str = "SELECT order_id FROM orders\n"
16+
17+
18+
def fake_run_engine(*, stderr_by_engine: dict[str, str]) -> Callable[..., EngineRun]:
19+
"""Return a `run_engine` that leaves one compiled test and prints the engine's stderr."""
20+
21+
def run_engine(
22+
*,
23+
project: CorpusProject,
24+
source_dir: Path,
25+
case_dir: Path,
26+
engine: str,
27+
side: str,
28+
options: DifferentialOptions,
29+
) -> EngineRun:
30+
del project, source_dir, side, options
31+
tests_dir: Path = case_dir / engine / "target" / "compiled" / "tests" / "orders"
32+
tests_dir.mkdir(parents=True)
33+
_ = (tests_dir / "test_orders.sql").write_text(ORDERS_TEST_SQL, encoding="utf-8")
34+
_ = (case_dir / engine / "target" / "sql-test-artifacts.json").write_text(
35+
"{}", encoding="utf-8"
36+
)
37+
return EngineRun(
38+
engine=engine,
39+
outcomes=(
40+
CommandOutcome(
41+
label="compile-warm",
42+
exit_code=0,
43+
stdout="{}",
44+
stderr=stderr_by_engine[engine],
45+
),
46+
),
47+
compiled={},
48+
manifest=None,
49+
dag=None,
50+
)
51+
52+
return run_engine
53+
54+
55+
def first_stat_paths(*, evidence_dir: Path, engines: tuple[str, ...]) -> list[str]:
56+
"""The first compiled-test path each engine's stat listing records."""
57+
58+
return [
59+
(evidence_dir / "example__orders" / engine / "compiled-tests-stat.tsv")
60+
.read_text(encoding="utf-8")
61+
.splitlines()[1]
62+
.split("\t")[0]
63+
for engine in engines
64+
]
65+
66+
67+
def kept_files(root: Path) -> tuple[str, ...]:
68+
"""Every file below `root` as sorted relative POSIX paths."""
69+
70+
return tuple(
71+
sorted(path.relative_to(root).as_posix() for path in filter(Path.is_file, root.rglob("*")))
72+
)
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
"""A differing project keeps both engines' target, stderr and compiled-test stats."""
2+
3+
from __future__ import annotations
4+
5+
from pathlib import Path
6+
7+
import pytest
8+
9+
from scripts.compiler_differential._helpers.comparing import project_comparison
10+
from scripts.compiler_differential.models import (
11+
CorpusProject,
12+
DifferentialCommand,
13+
DifferentialOptions,
14+
ExpectedOutcome,
15+
ProjectComparison,
16+
)
17+
from tests.unit.scripts.compiler_differential._helpers.running._test_types import (
18+
FailureEvidenceTestCase,
19+
)
20+
from tests.unit.scripts.compiler_differential._helpers.running.helpers import (
21+
fake_run_engine,
22+
first_stat_paths,
23+
kept_files,
24+
)
25+
26+
27+
@pytest.mark.parametrize(
28+
"test_case",
29+
[
30+
FailureEvidenceTestCase(
31+
description="differing warm stderr keeps both engines' evidence",
32+
stderr_by_engine={
33+
"python": "Semantic checks were partial for 1 models:\n",
34+
"native-preview": "SQL test planning START\n",
35+
},
36+
expected_files=(
37+
"example__orders/native-preview/0-compile-warm.stderr",
38+
"example__orders/native-preview/compiled-tests-stat.tsv",
39+
"example__orders/native-preview/target/compiled/tests/orders/test_orders.sql",
40+
"example__orders/native-preview/target/sql-test-artifacts.json",
41+
"example__orders/python/0-compile-warm.stderr",
42+
"example__orders/python/compiled-tests-stat.tsv",
43+
"example__orders/python/target/compiled/tests/orders/test_orders.sql",
44+
"example__orders/python/target/sql-test-artifacts.json",
45+
),
46+
expected_evidence_engines=("native-preview", "python"),
47+
),
48+
FailureEvidenceTestCase(
49+
description="identical runs keep nothing",
50+
stderr_by_engine={"python": "done\n", "native-preview": "done\n"},
51+
expected_files=(),
52+
expected_evidence_engines=(),
53+
),
54+
],
55+
ids=lambda case: case.description,
56+
)
57+
def test_given_engine_runs_when_comparing_then_only_differing_projects_keep_evidence(
58+
test_case: FailureEvidenceTestCase,
59+
tmp_path: Path,
60+
monkeypatch: pytest.MonkeyPatch,
61+
) -> None:
62+
monkeypatch.setattr(
63+
project_comparison,
64+
"run_engine",
65+
fake_run_engine(stderr_by_engine=test_case.stderr_by_engine),
66+
)
67+
evidence_dir: Path = tmp_path / "evidence"
68+
69+
comparison: ProjectComparison = project_comparison.compare_project(
70+
project=CorpusProject(
71+
name="example/orders",
72+
commands=(DifferentialCommand(label="compile-warm", arguments=("compile",)),),
73+
expected=ExpectedOutcome(),
74+
source_dir=tmp_path,
75+
),
76+
options=DifferentialOptions(
77+
engines=("python", "native-preview"),
78+
work_dir=tmp_path / "work",
79+
jobs=1,
80+
stage_captures=False,
81+
python=Path("python"),
82+
engine_environment={},
83+
evidence_dir=evidence_dir,
84+
),
85+
)
86+
stats: list[str] = first_stat_paths(
87+
evidence_dir=evidence_dir, engines=test_case.expected_evidence_engines
88+
)
89+
90+
assert (bool(comparison.differences), kept_files(evidence_dir), stats) == (
91+
bool(test_case.expected_files),
92+
test_case.expected_files,
93+
["orders/test_orders.sql"] * len(test_case.expected_evidence_engines),
94+
)

0 commit comments

Comments
 (0)