|
| 1 | +"""A differing project keeps both engines' target, stderr and compiled-test stats.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +from pathlib import Path |
| 6 | + |
| 7 | +import pytest |
| 8 | + |
| 9 | +from scripts.compiler_differential._helpers.comparing import project_comparison |
| 10 | +from scripts.compiler_differential.models import ( |
| 11 | + CorpusProject, |
| 12 | + DifferentialCommand, |
| 13 | + DifferentialOptions, |
| 14 | + ExpectedOutcome, |
| 15 | + ProjectComparison, |
| 16 | +) |
| 17 | +from tests.unit.scripts.compiler_differential._helpers.running._test_types import ( |
| 18 | + FailureEvidenceTestCase, |
| 19 | +) |
| 20 | +from tests.unit.scripts.compiler_differential._helpers.running.helpers import ( |
| 21 | + fake_run_engine, |
| 22 | + first_stat_paths, |
| 23 | + kept_files, |
| 24 | +) |
| 25 | + |
| 26 | + |
| 27 | +@pytest.mark.parametrize( |
| 28 | + "test_case", |
| 29 | + [ |
| 30 | + FailureEvidenceTestCase( |
| 31 | + description="differing warm stderr keeps both engines' evidence", |
| 32 | + stderr_by_engine={ |
| 33 | + "python": "Semantic checks were partial for 1 models:\n", |
| 34 | + "native-preview": "SQL test planning START\n", |
| 35 | + }, |
| 36 | + expected_files=( |
| 37 | + "example__orders/native-preview/0-compile-warm.stderr", |
| 38 | + "example__orders/native-preview/compiled-tests-stat.tsv", |
| 39 | + "example__orders/native-preview/target/compiled/tests/orders/test_orders.sql", |
| 40 | + "example__orders/native-preview/target/sql-test-artifacts.json", |
| 41 | + "example__orders/python/0-compile-warm.stderr", |
| 42 | + "example__orders/python/compiled-tests-stat.tsv", |
| 43 | + "example__orders/python/target/compiled/tests/orders/test_orders.sql", |
| 44 | + "example__orders/python/target/sql-test-artifacts.json", |
| 45 | + ), |
| 46 | + expected_evidence_engines=("native-preview", "python"), |
| 47 | + ), |
| 48 | + FailureEvidenceTestCase( |
| 49 | + description="identical runs keep nothing", |
| 50 | + stderr_by_engine={"python": "done\n", "native-preview": "done\n"}, |
| 51 | + expected_files=(), |
| 52 | + expected_evidence_engines=(), |
| 53 | + ), |
| 54 | + ], |
| 55 | + ids=lambda case: case.description, |
| 56 | +) |
| 57 | +def test_given_engine_runs_when_comparing_then_only_differing_projects_keep_evidence( |
| 58 | + test_case: FailureEvidenceTestCase, |
| 59 | + tmp_path: Path, |
| 60 | + monkeypatch: pytest.MonkeyPatch, |
| 61 | +) -> None: |
| 62 | + monkeypatch.setattr( |
| 63 | + project_comparison, |
| 64 | + "run_engine", |
| 65 | + fake_run_engine(stderr_by_engine=test_case.stderr_by_engine), |
| 66 | + ) |
| 67 | + evidence_dir: Path = tmp_path / "evidence" |
| 68 | + |
| 69 | + comparison: ProjectComparison = project_comparison.compare_project( |
| 70 | + project=CorpusProject( |
| 71 | + name="example/orders", |
| 72 | + commands=(DifferentialCommand(label="compile-warm", arguments=("compile",)),), |
| 73 | + expected=ExpectedOutcome(), |
| 74 | + source_dir=tmp_path, |
| 75 | + ), |
| 76 | + options=DifferentialOptions( |
| 77 | + engines=("python", "native-preview"), |
| 78 | + work_dir=tmp_path / "work", |
| 79 | + jobs=1, |
| 80 | + stage_captures=False, |
| 81 | + python=Path("python"), |
| 82 | + engine_environment={}, |
| 83 | + evidence_dir=evidence_dir, |
| 84 | + ), |
| 85 | + ) |
| 86 | + stats: list[str] = first_stat_paths( |
| 87 | + evidence_dir=evidence_dir, engines=test_case.expected_evidence_engines |
| 88 | + ) |
| 89 | + |
| 90 | + assert (bool(comparison.differences), kept_files(evidence_dir), stats) == ( |
| 91 | + bool(test_case.expected_files), |
| 92 | + test_case.expected_files, |
| 93 | + ["orders/test_orders.sql"] * len(test_case.expected_evidence_engines), |
| 94 | + ) |
0 commit comments