diff --git a/TODO.distribution/00-overview.md b/TODO.distribution/00-overview.md index d95ee52..e129bd2 100644 --- a/TODO.distribution/00-overview.md +++ b/TODO.distribution/00-overview.md @@ -71,6 +71,19 @@ raw.githubusercontent.com, never a branch head: - Adding a model: WO03 gate → merge entry → cut `index-vN+1` → runtimes pick it up on their next pin bump. +## Browser-tier capacity law (evidence record, 2026-08-29) + +From-scratch byte-level students collapse; pretrained init is the +variable that matters. The full arc in RESULTS.md: tiny 33M from-scratch +scored 83.08 DER (mojibake labels), retracted, the "clean rerun" silently +reused the poisoned snapshot (83.0797 - identical to 4 decimals), and the +true clean rerun (run-005, labels regenerated live) scored 74.68 vs the +3.07 gate. Label quality explains 8pp; capacity/init explains the rest. +Path to a sub-100MB browser artifact: SVD width-stitch DOWN from +pretrained ByT5-small (run-006-stitched, ~27M params = ~27MB int8), not +from-scratch training. Client stack already ships download-once Cache +API persistence (interscript@4.1.0). + ## File index - `00-overview.md` — this file diff --git a/src/gpu/distill_specs.yaml b/src/gpu/distill_specs.yaml index d64597d..87ae3f7 100644 --- a/src/gpu/distill_specs.yaml +++ b/src/gpu/distill_specs.yaml @@ -134,6 +134,33 @@ ara-diac-small-2: labels_file: teacher_labels_r7.jsonl mode: sequence note: 'r7 teacher + Muon; E4 gate: beat the shipped 8.259 by >= 2pp' +ara-diac-tiny-stitched: + teacher: rababa_arabic_byt5/run-006-morph/best + teacher_volume: rababa + out_volume: secryst + student_init: google/byt5-small + student_config: + d_model: 384 + d_ff: 928 + num_heads: 6 + enc_layers: 12 + dec_layers: 4 + feed_forward_proj: gated-gelu + train: r5-units/domain.txt + train_extra: + - r5-units/replay.txt + unit_limits: + - 8000 + - 4000 + max_len: 1450 + label_beams: '1' + out: rababa_arabic_distill_tiny/run-006-stitched + labels_file: teacher_labels_v2.jsonl + labels_complete: 'true' + mode: sequence + note: init-variable rung - same d384 width as the collapsed run-005 + (scratch, 74.68 DER) but SVD-stitched from pretrained ByT5-small; + gate <= 3.07 windowed DER-CE ara-diac-tiny: teacher: rababa_arabic_byt5/run-006-morph/best teacher_volume: rababa diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index a9ecbf5..c597308 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -88,6 +88,56 @@ def resolve_spec(spec: dict) -> dict: "best": str(Path(out_root) / "best"), } + +def svd_stitch_state(wide: dict, narrow: dict) -> dict: + """Closed-form width bridge: project pretrained wide weights into a + narrow state dict, preserving the top singular subspace per tensor + (microkimi protocol). 2-D: W' = U_o^T W V_i over leading singular + directions; 1-D: leading slice; N-D (head-count tensors): + leading-dimension slices.""" + import torch + + out = {} + for name, tgt in narrow.items(): + src = wide.get(name) + if src is None or tuple(src.shape) == tuple(tgt.shape): + out[name] = (src if src is not None else tgt).clone() + continue + if src.dim() == 2: + o, i = src.shape + o2, i2 = tgt.shape + w = src.float() + u, _, vh = torch.linalg.svd(w, full_matrices=False) + if o2 < o: + w = u[:, :o2].T @ w + if i2 < i: + w = w @ vh[:i2, :].T + if o2 > o or i2 > i: + padded = torch.zeros((o2, i2), dtype=w.dtype) + padded[: min(o, o2), : min(i, i2)] = w[: min(o, o2), : min(i, i2)] + w = padded + out[name] = w.to(tgt.dtype) + elif src.dim() == 1: + out[name] = src[: tgt.shape[0]].clone().to(tgt.dtype) + else: + out[name] = src[tuple(slice(0, s) for s in tgt.shape)].clone().to(tgt.dtype) + return out + + +def _maybe_stitch(spec_id: str, spec: dict, student) -> None: + """When a custom-width student also names a pretrained init, bridge + the pretrained weights down instead of random init (the capacity + law says init is the variable that matters — ara-diac-tiny run-005).""" + if not spec.get("student_init"): + return + from transformers import AutoModelForSeq2SeqLM + + pretrained = AutoModelForSeq2SeqLM.from_pretrained(spec["student_init"]) + print(f"[{spec_id}] svd-stitching from {spec['student_init']}", flush=True) + student.load_state_dict(svd_stitch_state(pretrained.state_dict(), student.state_dict())) + del pretrained + + def _ensure_src_path() -> None: # Modal copies the entry file to /root/.py while the repo image # sits at /root/interscript-ml — cover both layouts before importing @@ -568,11 +618,12 @@ def distill_sequence(spec_id: str, epochs: int = 3) -> dict: num_decoder_layers=cfg.get("dec_layers", 8), num_heads=cfg.get("num_heads", 6), dropout_rate=0.1, - feed_forward_proj="relu", + feed_forward_proj=cfg.get("feed_forward_proj", "relu"), decoder_start_token_id=0, relative_attention_max_distance=128, ) student = T5ForConditionalGeneration(config) + _maybe_stitch(spec_id, spec, student) n_params = sum(q.numel() for q in student.parameters()) / 1e6 print(f"[{spec_id}] tiny student: {n_params:.1f}M params", flush=True) else: @@ -1125,11 +1176,12 @@ def distill_microkimi(spec_id: str, epochs: int = 3, calib_batches: int = 64, num_decoder_layers=cfg.get("dec_layers", 8), num_heads=cfg.get("num_heads", 6), dropout_rate=0.1, - feed_forward_proj="relu", + feed_forward_proj=cfg.get("feed_forward_proj", "relu"), decoder_start_token_id=0, relative_attention_max_distance=128, ) student = T5ForConditionalGeneration(config) + _maybe_stitch(spec_id, spec, student) else: student = AutoModelForSeq2SeqLM.from_pretrained(spec["student_init"]) student.to("cuda").train() diff --git a/tests/test_stitch.py b/tests/test_stitch.py new file mode 100644 index 0000000..5939d4a --- /dev/null +++ b/tests/test_stitch.py @@ -0,0 +1,49 @@ +"""SVD width-stitch: project pretrained wide T5 weights into a narrow +config, preserving the top singular subspace (the microkimi +closed-form bridge).""" + +import torch + +from src.gpu.modal_distill import svd_stitch_state + + +def _wide_narrow(): + from transformers import T5Config, T5ForConditionalGeneration + + torch.manual_seed(0) + wide = T5ForConditionalGeneration( + T5Config(vocab_size=384, d_model=64, d_ff=128, d_kv=16, num_layers=2, + num_decoder_layers=2, feed_forward_proj="relu", + decoder_start_token_id=0, eos_token_id=1) + ) + narrow = T5ForConditionalGeneration( + T5Config(vocab_size=384, d_model=32, d_ff=64, d_kv=16, num_layers=2, + num_decoder_layers=2, feed_forward_proj="relu", + decoder_start_token_id=0, eos_token_id=1) + ) + return wide, narrow + + +def test_stitch_fills_every_parameter_with_target_shapes(): + wide, narrow = _wide_narrow() + wide_sd = wide.state_dict() + filled = svd_stitch_state(wide_sd, narrow.state_dict()) + for name, target in narrow.state_dict().items(): + assert name in filled, name + assert filled[name].shape == target.shape, name + assert torch.isfinite(filled[name]).all(), name + + +def test_stitch_preserves_top_subspace_action(): + wide, narrow = _wide_narrow() + wide_sd = wide.state_dict() + filled = svd_stitch_state(wide_sd, narrow.state_dict()) + w = wide_sd["encoder.block.0.layer.0.SelfAttention.q.weight"] # (128, 64) + w2 = filled["encoder.block.0.layer.0.SelfAttention.q.weight"] # (128, 32) + _, _, vh = torch.linalg.svd(w, full_matrices=False) + V = vh[:32, :].T # top-32 right singular vectors, orthonormal columns + torch.manual_seed(1) + z = torch.randn(32) + # for inputs in the kept right subspace (x = V z), the stitched + # layer reproduces the wide layer's action exactly + assert torch.allclose(w2 @ z, w @ (V @ z), atol=1e-4)