From 2f3c0f1f5ced0519eb92fc83f9c7f161bd9825e1 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 12:00:37 +0800 Subject: [PATCH 1/3] docs: ara-tiny clean-label re-run restores the capacity verdict (82.87% DER, immediate-EOS failure mode) --- docs/RESULTS.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 79cf78c..30b327d 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -144,11 +144,13 @@ reproducing its documented tier on this replication. Gate ≤ teacher + 0.5pp: the student misses by two orders of magnitude. -**RETRACTION (2026-08-24):** this verdict is CONFOUNDED — every Arabic -label generated before the byt5 `decode_joined` fix was mojibake -(double-encoded targets); both Arabic students trained on corrupted -labels, and their identical DER scores are the bare-text constant, not -a capacity result. The numbers stand as measured but the capacity -conclusion for Arabic is UNPROVEN pending a clean-label re-run. The -Thai tiny verdict is unaffected (umt5/sentencepiece labels were -byte-exact); the pretrained-backbone law rests on Thai evidence. +**RETRACTION (2026-08-24) and RESTORATION (same day):** the original +verdict was retracted when the labels proved mojibake (the byt5 +decode_joined bug); the clean-label re-run restores it — 33M student, +11,792 byte-exact r6 labels, train CE 1.55, windowed DER-CE **82.87%** +vs the teacher's 1.32% on the same 300-paragraph harness. The failure +mode is total: the trained student emits EOS immediately on free +running (empty output; it fits the training set under teacher forcing +but cannot sustain generation). The capacity conclusion for Arabic is +now UNCONFOUNDED and matches Thai: sub-100M from-scratch byte students +do not generalize; a pretrained backbone is non-negotiable. From 3ad0cc9dbc982eda9d8461d53864babaa716426b Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 12:11:08 +0800 Subject: [PATCH 2/3] fix(distill): step-checkpoint resume requires a matching labels digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today's clean-label run resumed from step-10500 of yesterday's poisoned run — 500 clean steps on a converged mojibake model, giving a model that scores clean labels at 1.33 loss yet generates mojibake. Each step checkpoint now records the sha of the labels file it was trained on; resume ignores checkpoints from a different labels lineage. --- src/gpu/modal_distill.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 76a11d4..c4e84b0 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -399,6 +399,7 @@ def val_loss() -> float: if step % save_every == 0: ck = out_root / f"step-{step}" ck.mkdir(exist_ok=True) + (ck / "labels.sha").write_text(labels_digest) torch.save(student.state_dict(), ck / "student.pt") torch.save(optimizer.state_dict(), ck / "optim.pt") CHECKPOINTS.commit() @@ -995,7 +996,20 @@ def __getitem__(self, i): save_every = 500 step = 0 - ckpts = sorted(out_root.glob("step-*"), key=lambda p: int(p.name.split("-")[1])) + import hashlib + + labels_digest = hashlib.sha256( + teacher_labels_path.read_bytes() + ).hexdigest()[:12] if teacher_labels_path.exists() else "none" + + def _usable(ck: Path) -> bool: + marker = ck / "labels.sha" + return marker.exists() and marker.read_text().strip() == labels_digest + + ckpts = sorted( + (c for c in out_root.glob("step-*") if _usable(c)), + key=lambda p: int(p.name.split("-")[1]), + ) if ckpts: student.load_state_dict( torch.load(ckpts[-1] / "student.pt", map_location="cpu", weights_only=True) @@ -1028,6 +1042,7 @@ def __getitem__(self, i): if step % save_every == 0: ck = out_root / f"step-{step}" ck.mkdir(exist_ok=True) + (ck / "labels.sha").write_text(labels_digest) torch.save(student.state_dict(), ck / "student.pt") torch.save(optimizer.state_dict(), ck / "optim.pt") CHECKPOINTS.commit() From babc07fe7ca8f569e2aa3119a2bf87bd41e243fe Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Mon, 24 Aug 2026 12:17:48 +0800 Subject: [PATCH 3/3] fix: labels.sha write belongs to distill_sequence only (logit-KD has no labels file) --- src/gpu/modal_distill.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index c4e84b0..7899372 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -399,7 +399,6 @@ def val_loss() -> float: if step % save_every == 0: ck = out_root / f"step-{step}" ck.mkdir(exist_ok=True) - (ck / "labels.sha").write_text(labels_digest) torch.save(student.state_dict(), ck / "student.pt") torch.save(optimizer.state_dict(), ck / "optim.pt") CHECKPOINTS.commit()