feat(distill): SVD width-stitch + ara-diac-tiny-stitched rung

TODO.distribution/00-overview.md

@@ -71,6 +71,19 @@ raw.githubusercontent.com, never a branch head:
 - Adding a model: WO03 gate → merge entry → cut `index-vN+1` →
   runtimes pick it up on their next pin bump.
 
+## Browser-tier capacity law (evidence record, 2026-08-29)
+
+From-scratch byte-level students collapse; pretrained init is the
+variable that matters. The full arc in RESULTS.md: tiny 33M from-scratch
+scored 83.08 DER (mojibake labels), retracted, the "clean rerun" silently
+reused the poisoned snapshot (83.0797 - identical to 4 decimals), and the
+true clean rerun (run-005, labels regenerated live) scored 74.68 vs the
+3.07 gate. Label quality explains 8pp; capacity/init explains the rest.
+Path to a sub-100MB browser artifact: SVD width-stitch DOWN from
+pretrained ByT5-small (run-006-stitched, ~27M params = ~27MB int8), not
+from-scratch training. Client stack already ships download-once Cache
+API persistence (interscript@4.1.0).
+
 ## File index
 
 - `00-overview.md` — this file

src/gpu/distill_specs.yaml

@@ -134,6 +134,33 @@ ara-diac-small-2:
   labels_file: teacher_labels_r7.jsonl
   mode: sequence
   note: 'r7 teacher + Muon; E4 gate: beat the shipped 8.259 by >= 2pp'
+ara-diac-tiny-stitched:
+  teacher: rababa_arabic_byt5/run-006-morph/best
+  teacher_volume: rababa
+  out_volume: secryst
+  student_init: google/byt5-small
+  student_config:
+    d_model: 384
+    d_ff: 928
+    num_heads: 6
+    enc_layers: 12
+    dec_layers: 4
+    feed_forward_proj: gated-gelu
+  train: r5-units/domain.txt
+  train_extra:
+  - r5-units/replay.txt
+  unit_limits:
+  - 8000
+  - 4000
+  max_len: 1450
+  label_beams: '1'
+  out: rababa_arabic_distill_tiny/run-006-stitched
+  labels_file: teacher_labels_v2.jsonl
+  labels_complete: 'true'
+  mode: sequence
+  note: init-variable rung - same d384 width as the collapsed run-005
+    (scratch, 74.68 DER) but SVD-stitched from pretrained ByT5-small;
+    gate <= 3.07 windowed DER-CE
 ara-diac-tiny:
   teacher: rababa_arabic_byt5/run-006-morph/best
   teacher_volume: rababa

src/gpu/modal_distill.py

@@ -88,6 +88,56 @@ def resolve_spec(spec: dict) -> dict:
         "best": str(Path(out_root) / "best"),
     }
 
+
+def svd_stitch_state(wide: dict, narrow: dict) -> dict:
+    """Closed-form width bridge: project pretrained wide weights into a
+    narrow state dict, preserving the top singular subspace per tensor
+    (microkimi protocol). 2-D: W' = U_o^T W V_i over leading singular
+    directions; 1-D: leading slice; N-D (head-count tensors):
+    leading-dimension slices."""
+    import torch
+
+    out = {}
+    for name, tgt in narrow.items():
+        src = wide.get(name)
+        if src is None or tuple(src.shape) == tuple(tgt.shape):
+            out[name] = (src if src is not None else tgt).clone()
+            continue
+        if src.dim() == 2:
+            o, i = src.shape
+            o2, i2 = tgt.shape
+            w = src.float()
+            u, _, vh = torch.linalg.svd(w, full_matrices=False)
+            if o2 < o:
+                w = u[:, :o2].T @ w
+            if i2 < i:
+                w = w @ vh[:i2, :].T
+            if o2 > o or i2 > i:
+                padded = torch.zeros((o2, i2), dtype=w.dtype)
+                padded[: min(o, o2), : min(i, i2)] = w[: min(o, o2), : min(i, i2)]
+                w = padded
+            out[name] = w.to(tgt.dtype)
+        elif src.dim() == 1:
+            out[name] = src[: tgt.shape[0]].clone().to(tgt.dtype)
+        else:
+            out[name] = src[tuple(slice(0, s) for s in tgt.shape)].clone().to(tgt.dtype)
+    return out
+
+
+def _maybe_stitch(spec_id: str, spec: dict, student) -> None:
+    """When a custom-width student also names a pretrained init, bridge
+    the pretrained weights down instead of random init (the capacity
+    law says init is the variable that matters — ara-diac-tiny run-005)."""
+    if not spec.get("student_init"):
+        return
+    from transformers import AutoModelForSeq2SeqLM
+
+    pretrained = AutoModelForSeq2SeqLM.from_pretrained(spec["student_init"])
+    print(f"[{spec_id}] svd-stitching from {spec['student_init']}", flush=True)
+    student.load_state_dict(svd_stitch_state(pretrained.state_dict(), student.state_dict()))
+    del pretrained
+
+
 def _ensure_src_path() -> None:
     # Modal copies the entry file to /root/<name>.py while the repo image
     # sits at /root/interscript-ml — cover both layouts before importing
@@ -568,11 +618,12 @@ def distill_sequence(spec_id: str, epochs: int = 3) -> dict:
             num_decoder_layers=cfg.get("dec_layers", 8),
             num_heads=cfg.get("num_heads", 6),
             dropout_rate=0.1,
-            feed_forward_proj="relu",
+            feed_forward_proj=cfg.get("feed_forward_proj", "relu"),
             decoder_start_token_id=0,
             relative_attention_max_distance=128,
         )
         student = T5ForConditionalGeneration(config)
+        _maybe_stitch(spec_id, spec, student)
         n_params = sum(q.numel() for q in student.parameters()) / 1e6
         print(f"[{spec_id}] tiny student: {n_params:.1f}M params", flush=True)
     else:
@@ -1125,11 +1176,12 @@ def distill_microkimi(spec_id: str, epochs: int = 3, calib_batches: int = 64,
             num_decoder_layers=cfg.get("dec_layers", 8),
             num_heads=cfg.get("num_heads", 6),
             dropout_rate=0.1,
-            feed_forward_proj="relu",
+            feed_forward_proj=cfg.get("feed_forward_proj", "relu"),
             decoder_start_token_id=0,
             relative_attention_max_distance=128,
         )
         student = T5ForConditionalGeneration(config)
+        _maybe_stitch(spec_id, spec, student)
     else:
         student = AutoModelForSeq2SeqLM.from_pretrained(spec["student_init"])
     student.to("cuda").train()

tests/test_stitch.py

@@ -0,0 +1,49 @@
+"""SVD width-stitch: project pretrained wide T5 weights into a narrow
+config, preserving the top singular subspace (the microkimi
+closed-form bridge)."""
+
+import torch
+
+from src.gpu.modal_distill import svd_stitch_state
+
+
+def _wide_narrow():
+    from transformers import T5Config, T5ForConditionalGeneration
+
+    torch.manual_seed(0)
+    wide = T5ForConditionalGeneration(
+        T5Config(vocab_size=384, d_model=64, d_ff=128, d_kv=16, num_layers=2,
+                 num_decoder_layers=2, feed_forward_proj="relu",
+                 decoder_start_token_id=0, eos_token_id=1)
+    )
+    narrow = T5ForConditionalGeneration(
+        T5Config(vocab_size=384, d_model=32, d_ff=64, d_kv=16, num_layers=2,
+                 num_decoder_layers=2, feed_forward_proj="relu",
+                 decoder_start_token_id=0, eos_token_id=1)
+    )
+    return wide, narrow
+
+
+def test_stitch_fills_every_parameter_with_target_shapes():
+    wide, narrow = _wide_narrow()
+    wide_sd = wide.state_dict()
+    filled = svd_stitch_state(wide_sd, narrow.state_dict())
+    for name, target in narrow.state_dict().items():
+        assert name in filled, name
+        assert filled[name].shape == target.shape, name
+        assert torch.isfinite(filled[name]).all(), name
+
+
+def test_stitch_preserves_top_subspace_action():
+    wide, narrow = _wide_narrow()
+    wide_sd = wide.state_dict()
+    filled = svd_stitch_state(wide_sd, narrow.state_dict())
+    w = wide_sd["encoder.block.0.layer.0.SelfAttention.q.weight"]  # (128, 64)
+    w2 = filled["encoder.block.0.layer.0.SelfAttention.q.weight"]  # (128, 32)
+    _, _, vh = torch.linalg.svd(w, full_matrices=False)
+    V = vh[:32, :].T  # top-32 right singular vectors, orthonormal columns
+    torch.manual_seed(1)
+    z = torch.randn(32)
+    # for inputs in the kept right subspace (x = V z), the stitched
+    # layer reproduces the wide layer's action exactly
+    assert torch.allclose(w2 @ z, w @ (V @ z), atol=1e-4)