From eb6535a0e7700241d2188480b73e9f626045da10 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 7 Sep 2026 19:17:32 +0200 Subject: [PATCH] =?UTF-8?q?test(runtime):=20golden-v1=20matrix=20=E2=80=94?= =?UTF-8?q?=20every=20model,=20byte-identical?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GOLDEN_DIR points at a golden-v1 checkout; each file's stem is the model id resolved from the index, decoded, and compared row-by-row. Generated from the released zips (modal_golden_matrix.py). --- runtime/tests/test_model.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/runtime/tests/test_model.py b/runtime/tests/test_model.py index 36be14e..925796c 100644 --- a/runtime/tests/test_model.py +++ b/runtime/tests/test_model.py @@ -156,3 +156,25 @@ def test_golden_set_e2e() -> None: rows = [json.loads(line) for line in golden.read_text(encoding="utf-8").splitlines()] for row in rows: assert model.translate(row["input"], max_len=128) == row["output"], row["input"] + + +def test_golden_matrix() -> None: + """golden-v1 corpus: every model's released zip must reproduce its + golden rows byte-identically. Set GOLDEN_DIR (release checkout); + each golden file maps to a model id resolved from the index.""" + import glob + + golden_dir = os.environ.get("GOLDEN_DIR") + if not golden_dir: + pytest.skip("set GOLDEN_DIR to a golden-v1 checkout") + files = sorted(glob.glob(str(Path(golden_dir) / "*.jsonl"))) + assert files, "no golden files" + for path in files: + model_id = Path(path).stem + model = Model.load(model_id) + rows = [json.loads(line) for line in Path(path).read_text(encoding="utf-8").splitlines() if line.strip()] + assert rows, model_id + for row in rows: + got = model.translate(row["input"], max_len=max(256, 4 * len(row["input"]))) + assert got == row["output"], (model_id, row["input"]) + del model