diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 79cf78c..30b327d 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -144,11 +144,13 @@ reproducing its documented tier on this replication. Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude. -**RETRACTION (2026-08-24):** this verdict is CONFOUNDED — every Arabic -label generated before the byt5 `decode_joined` fix was mojibake -(double-encoded targets); both Arabic students trained on corrupted -labels, and their identical DER scores are the bare-text constant, not -a capacity result. The numbers stand as measured but the capacity -conclusion for Arabic is UNPROVEN pending a clean-label re-run. The -Thai tiny verdict is unaffected (umt5/sentencepiece labels were -byte-exact); the pretrained-backbone law rests on Thai evidence. +**RETRACTION (2026-08-24) and RESTORATION (same day):** the original +verdict was retracted when the labels proved mojibake (the byt5 +decode_joined bug); the clean-label re-run restores it — 33M student, +11,792 byte-exact r6 labels, train CE 1.55, windowed DER-CE **82.87%** +vs the teacher's 1.32% on the same 300-paragraph harness. The failure +mode is total: the trained student emits EOS immediately on free +running (empty output; it fits the training set under teacher forcing +but cannot sustain generation). The capacity conclusion for Arabic is +now UNCONFOUNDED and matches Thai: sub-100M from-scratch byte students +do not generalize; a pretrained backbone is non-negotiable. diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 76a11d4..7899372 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -995,7 +995,20 @@ def __getitem__(self, i): save_every = 500 step = 0 - ckpts = sorted(out_root.glob("step-*"), key=lambda p: int(p.name.split("-")[1])) + import hashlib + + labels_digest = hashlib.sha256( + teacher_labels_path.read_bytes() + ).hexdigest()[:12] if teacher_labels_path.exists() else "none" + + def _usable(ck: Path) -> bool: + marker = ck / "labels.sha" + return marker.exists() and marker.read_text().strip() == labels_digest + + ckpts = sorted( + (c for c in out_root.glob("step-*") if _usable(c)), + key=lambda p: int(p.name.split("-")[1]), + ) if ckpts: student.load_state_dict( torch.load(ckpts[-1] / "student.pt", map_location="cpu", weights_only=True) @@ -1028,6 +1041,7 @@ def __getitem__(self, i): if step % save_every == 0: ck = out_root / f"step-{step}" ck.mkdir(exist_ok=True) + (ck / "labels.sha").write_text(labels_digest) torch.save(student.state_dict(), ck / "student.pt") torch.save(optimizer.state_dict(), ck / "optim.pt") CHECKPOINTS.commit()