Skip to content

Commit f9dbd8d

Browse files
Ronald Tseronaldtse
authored andcommitted
fix: re-apply explicit d_kv passthrough (regressed in the resumability commits)
ByT5 keeps d_kv=64 at any width; the d_model/heads derivation built inner dims of 1470 for the layerdrop student and crashed the bridge.
1 parent d0f2d9d commit f9dbd8d

1 file changed

Lines changed: 2 additions & 2 deletions

File tree

src/gpu/modal_distill.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -659,7 +659,7 @@ def distill_sequence(spec_id: str, epochs: int = 3) -> dict:
659659
vocab_size=259,
660660
d_model=cfg.get("d_model", 384),
661661
d_ff=cfg.get("d_ff", 1536),
662-
d_kv=cfg.get("d_model", 384) // cfg.get("num_heads", 6),
662+
d_kv=cfg.get("d_kv", cfg.get("d_model", 384) // cfg.get("num_heads", 6)),
663663
num_layers=cfg.get("enc_layers", 8),
664664
num_decoder_layers=cfg.get("dec_layers", 8),
665665
num_heads=cfg.get("num_heads", 6),
@@ -1217,7 +1217,7 @@ def distill_microkimi(spec_id: str, epochs: int = 3, calib_batches: int = 64,
12171217
vocab_size=259,
12181218
d_model=cfg.get("d_model", 384),
12191219
d_ff=cfg.get("d_ff", 1536),
1220-
d_kv=cfg.get("d_model", 384) // cfg.get("num_heads", 6),
1220+
d_kv=cfg.get("d_kv", cfg.get("d_model", 384) // cfg.get("num_heads", 6)),
12211221
num_layers=cfg.get("enc_layers", 8),
12221222
num_decoder_layers=cfg.get("dec_layers", 8),
12231223
num_heads=cfg.get("num_heads", 6),

0 commit comments

Comments
 (0)