From 92218cbabec4ae16dc852253f01fdaa7cc1f7143 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 25 Oct 2026 01:23:20 -0700 Subject: [PATCH 01/18] dasLLAMA: the MoE router's prefill GEMM takes the row nest The fp32 batch tile rounds in its lanes, and the router's top-k over tile-rounded logits parts from decode's at a near tie: gemma-4-26B-A4B Q4_0's CPU counting walk fell off its pinned tokens from the fourth. `matmul_batch_rows` is the nest as its own entry, under no knob and no override seam; `mm_fblob_rows` takes it at both router sites, so a prefill routes a token as decode would. `test_matmul_batch` holds the entry to the per-token matmul bit for bit at the router's shape, the tile apart as the control. `test_prefill_fp32` pins the nest for its bit-exact cells, as the Q8 cell pins the portable backend, and holds the shipped tile's last-position logits within 1e-4 of Nx forward, the two apart bit for bit as the control. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md | 7 ++- modules/dasLLAMA/dasllama/dasllama_blocks.das | 2 +- modules/dasLLAMA/dasllama/dasllama_common.das | 11 +++- modules/dasLLAMA/dasllama/dasllama_math.das | 29 ++++++++-- modules/dasLLAMA/dasllama/dasllama_ple.das | 2 +- modules/dasLLAMA/tests/test_matmul_batch.das | 57 +++++++++++++------ modules/dasLLAMA/tests/test_prefill.das | 33 ++++++++++- 7 files changed, 112 insertions(+), 29 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md index a03a9ead44..fa72221473 100644 --- a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -201,4 +201,9 @@ one-worker walk bit for bit); a batch and the same tokens one at a time do not, `test_matmul_batch.das` holds the fp32 path to the per-token dot within `F32_TILE_REL`. `g_f32_batch_tile_min_n` routes a dot shorter than it to `f32_dot_rows` whole; it ships at 1, so the tile serves every length, and `harness/f32_batch_race.das` is the lever that races the two constants -and the nest against the tile. +and the nest against the tile. `matmul_batch_rows` is the nest as its own entry, under no knob: a +batch whose outputs decide what decode decides on the GEMV's bits takes it - the MoE router's prefill +logits (`mm_fblob_rows`), whose top-k over tile-rounded logits parts from decode's at a near tie and +routes the token to another expert than decode would (`test_parity.das` holds the 26B-A4B's Q4_0 file +to its pinned tokens on the CPU backend; `test_matmul_batch.das` holds the entry to the per-token +matmul bit for bit at the router's shape). diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 38d90a504d..63baa6cb13 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -2943,7 +2943,7 @@ def private ffn_moe_prefill_grouped(t : Model; var s : Session; l : int64; npos // 1. route every position: one batched router GEMM, then the shared select per row; shexp gate dots ride along trace_tag(TRACE_TAG_ROUTE) let ts_route = prof_ticks() - mm_fblob_b(s.moe_rlog, t, t.router_off + l * ne * dim, s.xb_b, dim, ne, npos) + mm_fblob_rows(s.moe_rlog, t, t.router_off + l * ne * dim, s.xb_b, dim, ne, npos) prof_add("moe_router", ts_route) let ts_select = prof_ticks() // per-row bias+select+gate dot, threaded: serial loop was ~2-3ms every-lane-idle/layer here diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 987c10c4db..4fa526b9d2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -3743,13 +3743,18 @@ def mm_fblob(var y : array; t : Model; woff : int64; x : array; n, matmul(y, t.fblob, woff, x, n, d) } -// Batched fp32 matmul against the aux blob: Y [npos x d] token-major from X [npos x n] — the grouped -// MoE prefill's router. matmul_batch rounds in its tile's lanes (ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice), -// so a prefill's routing matches decode's to the lanes' rounding: a near-tie top-k can part the two. +// Batched fp32 matmul against the aux blob: Y [npos x d] token-major from X [npos x n], through the +// tile walk, which rounds in its lanes (ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice). def mm_fblob_b(var y : array; t : Model; woff : int64; x : array; n, d, npos : int64) { matmul_batch(y, t.fblob, woff, x, n, d, npos) } +//! The same through the row nest: every output the per-token `mm_fblob`'s bits. The MoE router's +//! prefill takes it, so a prefill routes a token as decode would. +def mm_fblob_rows(var y : array; t : Model; woff : int64; x : array; n, d, npos : int64) { + matmul_batch_rows(y, t.fblob, woff, x, n, d, npos) +} + // rmsnorm with the weight at `woff` in the fp32 aux blob — straight to the tuned pointer core. def rms_at(var o : array | #; t : Model; woff : int64; x : array | #; size : int64) { unsafe { diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 91a8d19311..a6257bc96c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -759,7 +759,8 @@ var private g_f32_batch_x_block = 256l * 1024l def public set_f32_batch_x_block(bytes : int64) { g_f32_batch_x_block = max(bytes, 4l * 1024l) } def public get_f32_batch_x_block() : int64 => g_f32_batch_x_block //! Under this dot length the row nest (`f32_dot_rows`) serves instead of the tile. Shipped at 1, so -//! the tile serves every length; the setter is `harness/f32_batch_race.das`'s lever to the nest arm. +//! the tile serves every length; the setter is the lever to the nest arm (`harness/f32_batch_race.das`, +//! `tests/test_prefill.das`'s bit-exact cells). var private g_f32_batch_tile_min_n = 1l def public set_f32_batch_tile_min_n(n : int64) { g_f32_batch_tile_min_n = max(n, 1l) } def public get_f32_batch_tile_min_n() : int64 => g_f32_batch_tile_min_n @@ -769,16 +770,23 @@ def private f32_batch_token_block(n, ntok : int64) : int64 { return clamp((fit / 4l) * 4l, 4l, max(ntok, 4l)) } +//! The row nest over rows [0, d) by every token, the rows split across the lanes: every output the +//! per-token `dot`'s bits +def private batch_rows_core(var yp : float?; wp : float const?; xp : float const?; n, d, ntok : int64) { + var myp = yp + let rch = matmul_chunks(int(d), get_q8_batch_chunks_per_job(), n * d * ntok) + maybe_parallel_for(0, int(d), rch) $(rb, re) { + f32_dot_rows(myp, wp, xp, n, d, int64(rb), int64(re), 0l, ntok) + } +} + // The fp32 batch GEMM core: 1-D row split, except ROW-STARVED shapes (MoE router: d=n_expert=128, // grain 16 => 8 chunks idling half a 16-lane box) split tokens too — a flat (row-chunk × token-block) // domain. Disjoint stores, the same tile walk per (row range, token range) either way. def private matmul_batch_core(var yp : float?; wp : float const?; xp : float const?; n, d, ntok : int64) { var myp = yp - if (n < g_f32_batch_tile_min_n) { // the row nest: only the race reaches it at the shipped 1 - let rch0 = matmul_chunks(int(d), get_q8_batch_chunks_per_job(), n * d * ntok) - maybe_parallel_for(0, int(d), rch0) $(rb, re) { - f32_dot_rows(myp, wp, xp, n, d, int64(rb), int64(re), 0l, ntok) - } + if (n < g_f32_batch_tile_min_n) { // the row nest: the knob's arm, reached under the lever alone + batch_rows_core(yp, wp, xp, n, d, ntok) return } let row_tiles = (d + 3l) / 4l // the dispatch unit: every chunk edge sits on the tile lattice @@ -868,6 +876,15 @@ def matmul_batch(var y : array; w : array; woff : int64; x : array matmul_batch_at(y, unsafe(addr(w[woff])), x, n, d, ntok) } +//! The blob+offset batch through the row nest, no knob and no override seam: every output the +//! per-token `matmul`'s bits. A batch whose outputs decide something decode decides on the GEMV's +//! bits - the MoE router's top-k - takes it; the tile's lanes would part a near tie. +def matmul_batch_rows(var y : array; w : array; woff : int64; x : array; n, d, ntok : int64) { + unsafe { + batch_rows_core(addr(y[0]), addr(w[woff]), addr(x[0]), n, d, ntok) + } +} + //! Borrowed-plane form: the weights live in a prepared image, indexed from `woff`. def matmul_batch(var y : array; w : PlaneF; woff : int64; x : array; n, d, ntok : int64) { matmul_batch_at(y, plane_at(w, woff), x, n, d, ntok) diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index d1e98ee303..e48b12120c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -398,7 +398,7 @@ def private gemma4_moe_prefill_grouped(t : Model; var s : Session; l : int64; np trace_tag(TRACE_TAG_ROUTE) let ts_route = prof_ticks() gemma4_router(t, s.xb2_b, s.x_b, l, dim, npos) - mm_fblob_b(s.moe_rlog, t, t.router_off + l * ne * dim, s.xb2_b, dim, ne, npos) + mm_fblob_rows(s.moe_rlog, t, t.router_off + l * ne * dim, s.xb2_b, dim, ne, npos) for (p in range64(npos)) { moe_select(c, unsafe(addr(s.moe_rlog[p * ne])), s.moe_idx_b, s.moe_w_b, p * k) } diff --git a/modules/dasLLAMA/tests/test_matmul_batch.das b/modules/dasLLAMA/tests/test_matmul_batch.das index 83a93a503e..226cecda26 100644 --- a/modules/dasLLAMA/tests/test_matmul_batch.das +++ b/modules/dasLLAMA/tests/test_matmul_batch.das @@ -124,8 +124,29 @@ def test_matmul_batch_lanes(t : T?) { } } +//! Y [ntok x d] as the per-token `matmul` lands it, a token at a time: the nest cells' expectation +def private per_token_matmul(w, x : array; n, d, ntok : int64) : array { + var want : array + want |> resize(ntok * d) + var xt : array + xt |> resize(n) + var yt : array + yt |> resize(d) + for (tk in range64(ntok)) { + for (j in range64(n)) { + xt[j] = x[tk * n + j] + } + matmul(yt, w, xt, n, d) + for (i in range64(d)) { + want[tk * d + i] = yt[i] + } + } + return <- want +} + //! the race's nest arm, reached through the knob alone: every output the per-token `dot`'s bits, -//! where the tile at the shipped knob lands other bits (the control that the knob switched arms) +//! where the tile at the shipped knob lands other bits (the control that the knob switched arms); +//! then the nest as its own entry at the MoE router's shape, the tile at the shipped knob apart again [test] def test_matmul_batch_nest_arm(t : T?) { t |> run("fp32 batch: the row nest under the knob is the per-token matmul bit for bit") @(t : T?) { @@ -134,21 +155,7 @@ def test_matmul_batch_nest_arm(t : T?) { let ntok = 37l let w <- fillf(n * d) let x <- fillf(ntok * n) - var want : array - want |> resize(ntok * d) - var xt : array - xt |> resize(n) - var yt : array - yt |> resize(d) - for (tk in range64(ntok)) { - for (j in range64(n)) { - xt[j] = x[tk * n + j] - } - matmul(yt, w, xt, n, d) - for (i in range64(d)) { - want[tk * d + i] = yt[i] - } - } + let want <- per_token_matmul(w, x, n, d, ntok) let min_n_was = get_f32_batch_tile_min_n() set_f32_batch_tile_min_n(1l << 30) var nest_y <- nan_fill(ntok * d) @@ -163,6 +170,24 @@ def test_matmul_batch_nest_arm(t : T?) { } t |> success(log_exact("fp32 batch tile at the shipped knob", tile_y, want) > 0, "control: the tile at the shipped knob rounds apart from the per-token dot") } + t |> run("fp32 batch: the nest entry is the per-token matmul bit for bit at the router's shape") @(t : T?) { + let n = 2048l // a 128-expert router over a 2048 hidden, a 97-token prompt: a whole row tile set, a token tail + let d = 128l + let ntok = 97l + let w <- fillf(n * d) + let x <- fillf(ntok * n) + let want <- per_token_matmul(w, x, n, d, ntok) + var rows_y <- nan_fill(ntok * d) + with_shipped_knobs() { + matmul_batch_rows(rows_y, w, 0l, x, n, d, ntok) + } + t |> equal(log_exact("fp32 batch rows entry", rows_y, want), 0, "elements differing between the nest entry and the per-token matmul") + var tile_y <- nan_fill(ntok * d) + with_shipped_knobs() { + matmul_batch(tile_y, w, x, n, d, ntok) + } + t |> success(log_exact("fp32 batch tile at the router's shape", tile_y, want) > 0, "control: the tile at the shipped knob rounds apart from the per-token dot at this shape") + } } // Q8·Q8: same, with weights quantized once and the activation quantized per token. diff --git a/modules/dasLLAMA/tests/test_prefill.das b/modules/dasLLAMA/tests/test_prefill.das index 6388c6a7f1..43cc66f7d8 100644 --- a/modules/dasLLAMA/tests/test_prefill.das +++ b/modules/dasLLAMA/tests/test_prefill.das @@ -17,14 +17,41 @@ require daslib/jobque_boost require daslib/fio require daslib/linq // sequence_equal: the logits rows and the codec-encoded KV byte blobs compare element for element (byte equality IS bit-exactness for a same-dtype pair) require dasllama/dasllama_math // with_dasllama_jobque_ + the runtime knob setters (engine-level, not re-exported) +require _compares // mismatch_rel, logits_maxdiff: the tile arm's bar // forward_prefill / forward_batch must be BIT-FOR-BIT identical to calling forward() once per // position: same last-position logits AND same KV cache — both for a from-scratch prompt and for a -// prefill that continues an already-populated cache (start_pos > 0, the multi-turn-chat path). +// prefill that continues an already-populated cache (start_pos > 0, the multi-turn-chat path). The +// fp32 model's batched GEMMs ride the tile walk, which rounds in its lanes +// (ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice), so its bit-exact cells pin the row nest - the +// same dot on both paths, as the Q8 cell's portable pin is - and one tile cell holds the shipped walk's +// last-position logits within PREFILL_TILE_REL of Nx forward. // Needs llama2.c's stories15M.bin (gitignored, 60MB); skips cleanly when absent. // override the machine-local llama2.c checkout with DASLLAMA_LLAMA2C_DIR (e.g. on the x64 handoff box) let MODEL_PATH = path_join(llama2c_dir(), "stories15M.bin") let TOKENIZER_PATH = path_join(llama2c_dir(), "tokenizer.bin") +let PREFILL_TILE_REL = 1.0e-4 // ten times the 1e-5 six layers of the tile's lane rounding put on the 15M model's logits + +// the shipped tile walk: forward_batch lands other bits than Nx forward (the control that the arm left +// the nest), the last-position logits within the tile's rounding +def check_prefill_tile(t : T?; tr : Model; prompt : array) { + let c = tr.config + let n = long_length(prompt) + with_dasllama_jobque_() { + set_prefill_attn_mode(AttnPrefillMode.classic) + var sr <- make_run_state(c) + for (p in range64(n)) { + forward(tr, sr, prompt[p], p) + } + var sp <- make_run_state(c) + forward_batch(tr, sp, prompt, n, 0l) + to_log(LOG_INFO, "tile prefill: last-position logits off Nx forward by {logits_maxdiff(sp.logits, sr.logits)} over logits up to {logits_maxabs(sr.logits)}\n") + t |> equal(mismatch_rel("tile prefill logits", sp.logits, sr.logits, PREFILL_TILE_REL), 0, "tile walk: last-position logits within {PREFILL_TILE_REL} of Nx forward") + t |> success(!sequence_equal(sr.logits, sp.logits), "control: the tile walk lands other bits than Nx forward") + delete sr + delete sp + } +} // forward_batch over the whole prompt (one batched pass from position 0) == N× forward. def check_prefill(t : T?; tr : Model; prompt : array; mode : AttnPrefillMode) { @@ -83,9 +110,13 @@ def test_prefill_fp32(t : T?) { var tk <- load_tokenizer(TOKENIZER_PATH, tr.config.vocab_size) let prompt <- encode(tk, "Once upon a time, there was a little girl", true, false) let long_prompt <- [for (i in range64(200l)); i % 200l + 1l] + let min_n_was = get_f32_batch_tile_min_n() + set_f32_batch_tile_min_n(1l << 30) // the row nest: the per-token dot on both paths check_prefill(t, tr, prompt, AttnPrefillMode.classic) check_continuation(t, tr, prompt, 4l, AttnPrefillMode.classic) check_prefill(t, tr, long_prompt, AttnPrefillMode.classic) // 200 tok trips g_attn_par_threshold (threaded arm) + set_f32_batch_tile_min_n(min_n_was) + check_prefill_tile(t, tr, prompt) delete tk delete tr } From f4429de7d7f3da4da2bbff3b8df0b9ffa48da7a5 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 25 Oct 2026 01:23:20 -0700 Subject: [PATCH 02/18] dasLLAMA tests: the depth knob's last round is the budget's to clip At depth 3 over a 32-token walk the eighth round drafts two: the budget has three tokens left. The cell expects every round at the knob's depth but a shorter last one. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/tests/CLAUDE.md | 3 ++- modules/dasLLAMA/tests/test_mtp_gemma_drafter.das | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index 2e28e29117..8c99f1cb2c 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -236,7 +236,8 @@ arm's gates and the served arms' GPU home: the file's entry below), its arms: ringed key run at the wrap, the trunk's pick alike), the drafts and the streams token for token, the ringed run's accepted drafts within a tenth of the flat run's; - `mtp-gdraft-depth` (served) - the counting walk at depths 3 and 1 token for token the plain scheduler's, every round - drafting the knob's depth across a 64-row page (depth 1's one a round the control), then four reasoning budgets at depth 3 + drafting the knob's depth across a 64-row page but the last round the 32-token budget clips (depth 1's one a round the + control), then four reasoning budgets at depth 3 whose forced close lands at the budget on the speculating stream; - `mtp-gdraft-joint` (served) - three streams on three regions at depths 1 and 2, a tick's verify carrying all three, the joint verifies counted; the control is one stream at depth 1 counting none; diff --git a/modules/dasLLAMA/tests/test_mtp_gemma_drafter.das b/modules/dasLLAMA/tests/test_mtp_gemma_drafter.das index a7d9c7ff2a..e3ec1f2117 100644 --- a/modules/dasLLAMA/tests/test_mtp_gemma_drafter.das +++ b/modules/dasLLAMA/tests/test_mtp_gemma_drafter.das @@ -933,7 +933,8 @@ def test_gemma_drafter_depth_knob(t : T?) { same_streams(t, trg, "gdraft depth {k}", plain, spec, 32l) let rounds = st1.rounds - st0.rounds t |> success(rounds > 0l, "depth {k}: the tick ran device rounds ({rounds})") - t |> equal(st1.drafted - st0.drafted, k * rounds, "depth {k}: every round drafted {k} tokens (the control is depth 1's one a round)") + let drafted = st1.drafted - st0.drafted + t |> success(drafted > k * (rounds - 1l) && drafted <= k * rounds, "depth {k}: every round drafted {k} tokens but a last round the 32-token budget clips ({drafted} over {rounds} rounds; the control is depth 1's one a round)") t |> success(spec.accepted > 0l, "depth {k}: drafts were accepted ({spec.accepted} of {spec.drafted})") to_log(LOG_INFO, "gdraft depth {k}: {spec.accepted} of {spec.drafted} drafts over {rounds} rounds, {spec.ticks} ticks for 32 tokens\n") delete spec From b55adcf1e7d4756fc45a22319687fef13bbbb2b8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 19:07:14 -0700 Subject: [PATCH 03/18] dasLLAMA: gelu_erf runs on the float lattice as a tuned loop kernel The erf GELU evaluated erfc in double through libm's exp, one element at a time, 8.6 ns an element a lane; the Xeon's Qwen3-ASR conv front spent about 1.3 s of its 2.9 s encode in it. The kernel now computes the same NR erfc fit over exp4's polynomial in float, every op arithmetic, so the loop vectorizes: 0.85 ns an element a lane on the M5, the fp64-series cells within their 1e-6 bars. The tuner races it as the 27th tuned kernel. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/dasllama/dasllama_math.das | 28 +++++++++ modules/dasLLAMA/dasllama/dasllama_tower.das | 26 +-------- modules/dasLLAMA/harness/tune_kernels.das | 60 +++++++++++++++++++- 3 files changed, 88 insertions(+), 26 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index a6257bc96c..97b8917815 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -2591,6 +2591,34 @@ def exp4(x : float4) : float4 { } } +//! gelu_erf(x) = 0.5 x (1 + erf(x / sqrt 2)) in place on the float lattice: erfc(|x| / sqrt 2) as the NR +//! rational fit (fractional error under 1.2e-7) over exp4's polynomial a lane, the sign picking 0.5 x erfc +//! or 0.5 x (2 - erfc) so both tails keep relative accuracy; all arithmetic, so the loop vectorizes. +[hint(unsafe_range_check, noalias = x)] +def template gelu_erf_template(var x : float ?; n : int64) : void { + for [tune = 1] (i in range64(n)) { + unsafe { + let v = x[i] + let z = abs(v) * 0.7071067811865476 + let t = 1.0 / (1.0 + 0.5 * z) + let a = max(-z * z - 1.26551223 + t * (1.00002368 + t * (0.37409196 + t * (0.09678418 + t * (-0.18628806 + + t * (0.27886807 + t * (-1.13520398 + t * (1.48851587 + t * (-0.82215223 + t * 0.17087277)))))))), -87.0) + let zz = mad(a, 1.4426950216293335, 12582912.0) + let nn = zz - 12582912.0 + let b = mad(nn, -1.428606765330187e-06, mad(nn, -0.693145751953125, a)) + let u = b * b + let p = mad(mad(mad(0.008247390389442444, b, 0.04189976677298546), u, + mad(0.16668395698070526, b, 0.4999912679195404)), u, 0.9999994039535522 * b) + let k = reinterpret((reinterpret(zz) << 23u) + 0x3f800000u) + let erfc = t * mad(p, k, k) + x[i] = v >= 0.0 ? 0.5 * v * (2.0 - erfc) : 0.5 * v * erfc + } + } +} + +[hint(unsafe_range_check, noalias = x), tuned] +def gelu_erf(var x : float?; n : int64) {} + //! Index of the max element, first-index-wins on ties — the greedy-decode argmax. Chunked team //! scan (serial 151-262k-vocab scan idled every lane 210-284us/token); combine folds candidates //! in index order with strict >, so the result is bit-identical to the serial loop for ANY split. diff --git a/modules/dasLLAMA/dasllama/dasllama_tower.das b/modules/dasLLAMA/dasllama/dasllama_tower.das index 38cf3dbd82..8b75db8d52 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tower.das +++ b/modules/dasLLAMA/dasllama/dasllama_tower.das @@ -90,30 +90,8 @@ def ln_block_regions(o : LayerOffs; d, ff : int64; var out : Q8Region[int(LN_BLO out[LN_GEMM_FC2] = Q8Region(woff = o.fc2_w, n = ff, rows = d) } -// erfc rational approximation (Numerical Recipes 6.2), |fractional error| < 1.2e-7 — -// float-level agreement with the reference erff. Double internally. -def private erf_nr(x : double) : double { - let z = abs(x) - let t = 1.0lf / (1.0lf + 0.5lf * z) - let erfc = t * exp(-z * z - 1.26551223lf + t * (1.00002368lf + t * (0.37409196lf + t * (0.09678418lf + - t * (-0.18628806lf + t * (0.27886807lf + t * (-1.13520398lf + t * (1.48851587lf + - t * (-0.82215223lf + t * 0.17087277lf))))))))) - return x >= 0.0lf ? 1.0lf - erfc : erfc - 1.0lf -} - -//! gelu_erf(x) = 0.5·x·(1 + erf(x/√2)) — the EXACT erf form, -//! not the tanh approximation ``gelu`` uses. Whisper's conv frontend and encoder FFN both -//! use this flavor. In place over the first ``size`` elements. -def gelu_erf(var x : float?; size : int64) { - let inv_sqrt2 = 0.7071067811865476lf - unsafe { - for (i in range64(size)) { - let v = double(x[i]) - x[i] = float(0.5lf * v * (1.0lf + erf_nr(v * inv_sqrt2))) - } - } -} - +//! gelu_erf(x) = 0.5·x·(1 + erf(x/√2)) — the erf form, not the tanh approximation ``gelu`` uses +//! (the kernel: dasllama_math); whisper's conv frontend and encoder FFN use it. In place over ``size``. def gelu_erf(var x : array | #; size : int64) { if (size <= 0l) return gelu_erf(unsafe(addr(x[0])), size) diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index dabf2a276f..0baaeb7088 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -80,6 +80,8 @@ struct GridRopeTab {} struct GridGemm {} [dasllama_grid(src = "f32_batch_tile_template")] struct GridF32BatchTile {} +[dasllama_grid(src = "gelu_erf_template")] +struct GridGeluErf {} [dasllama_grid(src = "dot_q8q8_laneq4x4_template")] struct GridLaneq {} [dasllama_grid(src = "dot_f16_template")] @@ -157,7 +159,7 @@ let TARGET_WINDOW_US = 20000.0lf // same window as tuner_noise's PROBE_WINDOW_ let MAX_REPS_SCALE = 32 // The outer progress denominator, needed before the first kernel runs so it cannot be derived // from the results. A mismatch only skews the bar, and main says so at the end. -let TUNED_KERNEL_COUNT = 26 +let TUNED_KERNEL_COUNT = 27 struct private KernelResult { kernel : string @@ -1066,6 +1068,60 @@ def bench_softmax() : string { return report("softmax", race) } +// ===== gelu_erf: in place over N, gated f64-relative against the erf series' GELU on a [-4, 4] spread. + +def private erf_series(x : double) : double { + var sum = 0.0lf + var term = x + var k = 0 + while (abs(term) > 1.0e-18lf && k < 200) { + sum += term / double(2 * k + 1) + k++ + term *= -x * x / double(k) + } + return sum * 2.0lf / sqrt(3.141592653589793lf) +} + +def bench_gelu_erf() : string { + let vs <- gelu_erf_template_variants() + var inscope d0, d : array + d0 |> resize(N) + d |> resize(N) + for (i in range64(N)) { + d0[i] = 4.0f * sin(float(i) * 0.11f) + } + var dp = unsafe(addr(d[0])) + var inscope want : array + want |> resize(N) + for (i in range64(N)) { + let x = double(d0[i]) + want[i] = 0.5lf * x * (1.0lf + erf_series(x * 0.7071067811865476lf)) + } + var sink = 0.0f + var inscope race <- race_variants(vs, $(i : int) : bool { + reset_stream(dp, d0) + let f = vs[i]._1 + invoke(f, dp, N) + var maxrel = 0.0lf + for (j in range64(N)) { + let rel = rel_err(double(d[j]), want[j]) + maxrel = worst_rel(maxrel, rel) + } + return maxrel <= TOL + }, $(_i : int) { + reset_stream(dp, d0) + }, $(i : int) { + let f = vs[i]._1 + for (_rep in range(reps_of(i))) { + invoke(f, dp, N) + } + }, $ { + sink += d[0] + }) + tune_detail(" checksum {sink}\n") + return report("gelu_erf", race) +} + // ===== rmsnorm: (var o, x, weight, size, eps) out-of-place — repeatable, no reset needed. def bench_rmsnorm() : string { @@ -1975,7 +2031,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; ("cvt_q8kv_to_f32", @@bench_cvt_q8kv_to_f32), ("quantize_q8kv_row", @@bench_quantize_q8kv_row), ("dot_mx4q8", @@bench_dot_mx4q8), ("quantize_q8_0_into_ptr", @@bench_quantize), ("quantize_q8_0_bs_into_ptr", @@bench_quantize_bs), ("rope_scaled_neox_tab", @@bench_rope_tab), - ("gemm_f32_uk_4x16", @@bench_gemm_tile), ("f32_batch_tile", @@bench_f32_batch_tile), + ("gemm_f32_uk_4x16", @@bench_gemm_tile), ("f32_batch_tile", @@bench_f32_batch_tile), ("gelu_erf", @@bench_gelu_erf), ("dot_q8q8_laneq4x4", @@bench_laneq4x4) ] let partialMint = !empty(tune_only_filter()) From 7e6af3a1ac3023b7a0a6ca12a578986f1d4fe7e2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 19:13:02 -0700 Subject: [PATCH 04/18] dasLLAMA: a bf16 weight plane takes the fp32 batch tile walk matmul_bf16_batch ran a dot_bf16 nest, one output a dot at two loads a FMA - the gemma-E per-layer projection and the bf16 tower planes paid it on every CPU prefill. The tile walk is now generic over the weight lattice: the same row-tile dispatch, token blocks and tails, with bf16_batch_tile widening each halfword by the exact shift inside the sixteen-accumulator loop, so the bf16 plane lands the f32 plane's rate (1146 against 1073 GFLOP/s at the E4B projection's pp512 shape on the M5) at half the weight bytes. Decode keeps the GEMV chunk shaper. The tuner races the tile as the 28th kernel; the GEMM test holds three bf16 shapes to the plain oracle over the widened weights. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md | 4 +- .../dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md | 3 +- modules/dasLLAMA/dasllama/dasllama_math.das | 87 ++++++++++++++----- modules/dasLLAMA/harness/tune_kernels.das | 56 +++++++++++- modules/dasLLAMA/tests/test_matmul_batch.das | 39 +++++++++ 5 files changed, 163 insertions(+), 26 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md index fa72221473..ebef0281ce 100644 --- a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -195,11 +195,11 @@ dispatch unit is a row tile, a token chunk is whole tiles, every token block (`f32_batch_token_block`: the X rows that fit the smaller of the box's q8 L2 budget and `g_f32_batch_x_block` bytes, rounded down to whole tiles, or the whole token range when it fits) starts on the lattice, and only the rows past the last whole row tile of d and the tokens past the -last whole token tile of ntok take `dot` (`f32_dot_rows`). Any row chunking, worker count or token +last whole token tile of ntok take `dot` (`batch_dot_rows`). Any row chunking, worker count or token split then lands the same bits (`test_matmul_batch.das`'s lanes cell holds the threaded walk to the one-worker walk bit for bit); a batch and the same tokens one at a time do not, and `test_matmul_batch.das` holds the fp32 path to the per-token dot within `F32_TILE_REL`. -`g_f32_batch_tile_min_n` routes a dot shorter than it to `f32_dot_rows` whole; it ships at 1, so the +`g_f32_batch_tile_min_n` routes a dot shorter than it to `batch_dot_rows` whole; it ships at 1, so the tile serves every length, and `harness/f32_batch_race.das` is the lever that races the two constants and the nest against the tile. `matmul_batch_rows` is the nest as its own entry, under no knob: a batch whose outputs decide what decode decides on the GEMV's bits takes it - the MoE router's prefill diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md index 53933b8981..0223a54243 100644 --- a/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md +++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md @@ -248,7 +248,8 @@ repack walks that plane's region alone (`shexp_plane_q8`). wrappers, dispatch shaping, and the array-sizing helpers every tier and the engine share (`reserve_resize`, `grow_resize`, `ensure_length`, `overwrite_resize`), the `[tuned]` fp32 loop-hint families (`dot`, `axpy`, the elementwise set, `f32_batch_tile`) and the fp32 batch walk - (`matmul_batch_core`, `f32_batch_block`); every other kernel lives in a tier file. + (`matmul_batch_core`, `batch_tile_block`, generic over the f32 and bf16 weight lattices - `bf16_batch_tile` + widens each halfword by the exact shift); every other kernel lives in a tier file. - **`dasllama_math_default.das`** - the portable backend, always registered, always correct, out-ranked by any platform tier - the per-32 rail's portable kernels and scalar leaves, which seat every backend's unset per-32 slot (`#per-32-rail`) - and the ONE body of every Q8*Q8 and diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 97b8917815..7928545482 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -710,12 +710,56 @@ def template f32_batch_tile_template(var yp : float ?; wp, xp : float const ?; n [hint(unsafe_range_check, noalias = yp, noalias = wp, noalias = xp), tuned(fallback = "vec8")] def f32_batch_tile(var yp : float?; wp, xp : float const?; n, d : int64) {} -//! The tile walk over rows [r0, r1) by tokens [t0, t1) in token blocks of `x_block_tok` +//! The f32 tile over a bf16 weight plane: the same sixteen accumulators, each weight widened by the +//! exact high-half shift as it is read, so a bf16 plane pays half the weight bytes for the same products. +[hint(unsafe_range_check, noalias = yp, noalias = wp, noalias = xp)] +def template bf16_batch_tile_template(var yp : float ?; wp : uint16 const ?; xp : float const ?; n, d : int64) : void { + var a00 = 0.0; var a01 = 0.0; var a02 = 0.0; var a03 = 0.0 + var a10 = 0.0; var a11 = 0.0; var a12 = 0.0; var a13 = 0.0 + var a20 = 0.0; var a21 = 0.0; var a22 = 0.0; var a23 = 0.0 + var a30 = 0.0; var a31 = 0.0; var a32 = 0.0; var a33 = 0.0 + let w1 = unsafe(wp + n); let w2 = unsafe(wp + 2l * n); let w3 = unsafe(wp + 3l * n) + let x1 = unsafe(xp + n); let x2 = unsafe(xp + 2l * n); let x3 = unsafe(xp + 3l * n) + for [tune = 1] (j in range64(n)) { + unsafe { + let xa = xp[j]; let xb = x1[j]; let xc = x2[j]; let xd = x3[j] + let wa = reinterpret(uint(wp[j]) << 16u) + a00 = mad(wa, xa, a00); a01 = mad(wa, xb, a01); a02 = mad(wa, xc, a02); a03 = mad(wa, xd, a03) + let wb = reinterpret(uint(w1[j]) << 16u) + a10 = mad(wb, xa, a10); a11 = mad(wb, xb, a11); a12 = mad(wb, xc, a12); a13 = mad(wb, xd, a13) + let wc = reinterpret(uint(w2[j]) << 16u) + a20 = mad(wc, xa, a20); a21 = mad(wc, xb, a21); a22 = mad(wc, xc, a22); a23 = mad(wc, xd, a23) + let wd = reinterpret(uint(w3[j]) << 16u) + a30 = mad(wd, xa, a30); a31 = mad(wd, xb, a31); a32 = mad(wd, xc, a32); a33 = mad(wd, xd, a33) + } + } + unsafe { + yp[0l] = a00; yp[1l] = a10; yp[2l] = a20; yp[3l] = a30 + yp[d] = a01; yp[d + 1l] = a11; yp[d + 2l] = a21; yp[d + 3l] = a31 + yp[2l * d] = a02; yp[2l * d + 1l] = a12; yp[2l * d + 2l] = a22; yp[2l * d + 3l] = a32 + yp[3l * d] = a03; yp[3l * d + 1l] = a13; yp[3l * d + 2l] = a23; yp[3l * d + 3l] = a33 + } +} +[hint(unsafe_range_check, noalias = yp, noalias = wp, noalias = xp), tuned(fallback = "vec8")] +def bf16_batch_tile(var yp : float?; wp : uint16 const?; xp : float const?; n, d : int64) {} + +//! the walk's tile and tail dot by weight lattice: the f32 plane's and the bf16 plane's +def private batch_tile(var yp : float?; wp : float const?; xp : float const?; n, d : int64) { + f32_batch_tile(yp, wp, xp, n, d) +} +def private batch_tile(var yp : float?; wp : uint16 const?; xp : float const?; n, d : int64) { + bf16_batch_tile(yp, wp, xp, n, d) +} +def private row_dot(wrow : float const?; xrow : float const?; n : int64) : float => dot(wrow, xrow, n) +def private row_dot(wrow : uint16 const?; xrow : float const?; n : int64) : float => dot_bf16(xrow, wrow, n) + +//! The tile walk over rows [r0, r1) by tokens [t0, t1) in token blocks of `x_block_tok`, over an f32 or a +//! bf16 weight plane [hint(noalias = yp, noalias = wp, noalias = xp), arch(at = "../ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice")] -def f32_batch_block(var yp : float?; wp, xp : float const?; n, d, r0, r1, t0, t1, x_block_tok : int64) { +def batch_tile_block(var yp : float?; wp : auto const?; xp : float const?; n, d, r0, r1, t0, t1, x_block_tok : int64) { let ra = min(r1, ((r0 + 3l) / 4l) * 4l) let r4 = ra + ((r1 - ra) / 4l) * 4l - f32_dot_rows(yp, wp, xp, n, d, r0, ra, t0, t1) + batch_dot_rows(yp, wp, xp, n, d, r0, ra, t0, t1) var tbeg = t0 while (tbeg < t1) { let tend = min(tbeg + x_block_tok, t1) @@ -726,14 +770,14 @@ def f32_batch_block(var yp : float?; wp, xp : float const?; n, d, r0, r1, t0, t1 let wrow = wp + i * n var tk = tbeg while (tk < t4) { - f32_batch_tile(yp + tk * d + i, wrow, xp + tk * n, n, d) + batch_tile(yp + tk * d + i, wrow, xp + tk * n, n, d) tk += 4l } } - f32_dot_rows(yp, wp, xp, n, d, i, i + 4l, t4, tend) + batch_dot_rows(yp, wp, xp, n, d, i, i + 4l, t4, tend) i += 4l } - f32_dot_rows(yp, wp, xp, n, d, r4, r1, tbeg, tend) + batch_dot_rows(yp, wp, xp, n, d, r4, r1, tbeg, tend) tbeg = tend } } @@ -741,12 +785,12 @@ def f32_batch_block(var yp : float?; wp, xp : float const?; n, d, r0, r1, t0, t1 //! Rows [r0, r1) by tokens [t0, t1), each output its own `dot`: the tile walk's tails and the row nest //! share it, and the outputs are independent, so the loop order moves no bit. [hint(noalias = yp, noalias = wp, noalias = xp), arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kernel-visibility-lifted-workers")] -def f32_dot_rows(var yp : float?; wp, xp : float const?; n, d, r0, r1, t0, t1 : int64) { +def batch_dot_rows(var yp : float?; wp : auto const?; xp : float const?; n, d, r0, r1, t0, t1 : int64) { unsafe { for (i in range64(r0, r1)) { let wrow = wp + i * n for (tk in range64(t0, t1)) { - yp[tk * d + i] = dot(wrow, xp + tk * n, n) + yp[tk * d + i] = row_dot(wrow, xp + tk * n, n) } } } @@ -758,7 +802,7 @@ def f32_dot_rows(var yp : float?; wp, xp : float const?; n, d, r0, r1, t0, t1 : var private g_f32_batch_x_block = 256l * 1024l def public set_f32_batch_x_block(bytes : int64) { g_f32_batch_x_block = max(bytes, 4l * 1024l) } def public get_f32_batch_x_block() : int64 => g_f32_batch_x_block -//! Under this dot length the row nest (`f32_dot_rows`) serves instead of the tile. Shipped at 1, so +//! Under this dot length the row nest (`batch_dot_rows`) serves instead of the tile. Shipped at 1, so //! the tile serves every length; the setter is the lever to the nest arm (`harness/f32_batch_race.das`, //! `tests/test_prefill.das`'s bit-exact cells). var private g_f32_batch_tile_min_n = 1l @@ -772,18 +816,18 @@ def private f32_batch_token_block(n, ntok : int64) : int64 { //! The row nest over rows [0, d) by every token, the rows split across the lanes: every output the //! per-token `dot`'s bits -def private batch_rows_core(var yp : float?; wp : float const?; xp : float const?; n, d, ntok : int64) { +def private batch_rows_core(var yp : float?; wp : auto const?; xp : float const?; n, d, ntok : int64) { var myp = yp let rch = matmul_chunks(int(d), get_q8_batch_chunks_per_job(), n * d * ntok) maybe_parallel_for(0, int(d), rch) $(rb, re) { - f32_dot_rows(myp, wp, xp, n, d, int64(rb), int64(re), 0l, ntok) + batch_dot_rows(myp, wp, xp, n, d, int64(rb), int64(re), 0l, ntok) } } // The fp32 batch GEMM core: 1-D row split, except ROW-STARVED shapes (MoE router: d=n_expert=128, // grain 16 => 8 chunks idling half a 16-lane box) split tokens too — a flat (row-chunk × token-block) // domain. Disjoint stores, the same tile walk per (row range, token range) either way. -def private matmul_batch_core(var yp : float?; wp : float const?; xp : float const?; n, d, ntok : int64) { +def private matmul_batch_core(var yp : float?; wp : auto const?; xp : float const?; n, d, ntok : int64) { var myp = yp if (n < g_f32_batch_tile_min_n) { // the row nest: the knob's arm, reached under the lever alone batch_rows_core(yp, wp, xp, n, d, ntok) @@ -796,7 +840,7 @@ def private matmul_batch_core(var yp : float?; wp : float const?; xp : float con let x_block_tok = f32_batch_token_block(n, ntok) if (tb <= 1l) { maybe_parallel_for(0, int(row_tiles), rch) $(rb, re) { - f32_batch_block(myp, wp, xp, n, d, int64(rb) * 4l, min(int64(re) * 4l, d), 0l, ntok, x_block_tok) + batch_tile_block(myp, wp, xp, n, d, int64(rb) * 4l, min(int64(re) * 4l, d), 0l, ntok, x_block_tok) } return } @@ -809,7 +853,7 @@ def private matmul_batch_core(var yp : float?; wp : float const?; xp : float con let r0 = min(rc * tiles_per * 4l, d) let r1 = min(r0 + tiles_per * 4l, d) let t1 = min(tc * tok_per + tok_per, ntok) - f32_batch_block(myp, wp, xp, n, d, r0, r1, tc * tok_per, t1, x_block_tok) + batch_tile_block(myp, wp, xp, n, d, r0, r1, tc * tok_per, t1, x_block_tok) } } } @@ -902,8 +946,8 @@ def matmul_batch(var y : array; w : PlaneF; woff : int64; x : PlaneF; n, } //! Batched matvec against a native-BF16 weight [d x n] at halfword offset woff: Y [ntok x d] -//! token-major. Bit-for-bit matmul_batch's fp32 expand at half the weight read (dot_bf16's widen -//! is exact). Decode (ntok == 1) takes the GEMV chunk shaper. +//! token-major, the fp32 tile walk over the halfword plane (the widen is exact, so the outputs are +//! matmul_batch's over the expanded plane). Decode (ntok == 1) takes the GEMV chunk shaper. def matmul_bf16_batch(var y : array; w : array; woff : int64; x : array; n, d, ntok : int64) { matmul_bf16_batch_at(y, unsafe(addr(w[woff])), x, n, d, ntok) } @@ -920,14 +964,15 @@ def private matmul_bf16_batch_at(var y : array; wp : uint16 const?; x : a if (g_fp_override_active && ntok >= g_fp_override_min_ntok && invoke(g_mm_bf16_batch_override, yp, wp, xp, n, d, ntok)) { return } - let nch = ntok == 1l ? matmul_chunks_gemv(int(d), 1, n * d) : matmul_chunks(int(d), get_q8_batch_chunks_per_job(), n * d * ntok) + if (ntok > 1l) { + matmul_batch_core(yp, wp, xp, n, d, ntok) + return + } + let nch = matmul_chunks_gemv(int(d), 1, n * d) maybe_parallel_for(0, int(d), nch) $(rb, re) { unsafe { for (i in range(rb, re)) { - let wrow = wp + int64(i) * n - for (tk in range64(ntok)) { - yp[tk * d + int64(i)] = dot_bf16(xp + tk * n, wrow, n) - } + yp[int64(i)] = dot_bf16(xp, wp + int64(i) * n, n) } } } diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 0baaeb7088..03b94fde2d 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -82,6 +82,8 @@ struct GridGemm {} struct GridF32BatchTile {} [dasllama_grid(src = "gelu_erf_template")] struct GridGeluErf {} +[dasllama_grid(src = "bf16_batch_tile_template")] +struct GridBf16BatchTile {} [dasllama_grid(src = "dot_q8q8_laneq4x4_template")] struct GridLaneq {} [dasllama_grid(src = "dot_f16_template")] @@ -159,7 +161,7 @@ let TARGET_WINDOW_US = 20000.0lf // same window as tuner_noise's PROBE_WINDOW_ let MAX_REPS_SCALE = 32 // The outer progress denominator, needed before the first kernel runs so it cannot be derived // from the results. A mismatch only skews the bar, and main says so at the end. -let TUNED_KERNEL_COUNT = 27 +let TUNED_KERNEL_COUNT = 28 struct private KernelResult { kernel : string @@ -1068,6 +1070,56 @@ def bench_softmax() : string { return report("softmax", race) } +// ===== bf16_batch_tile: the f32 tile over a bf16 weight plane - the same gate, the weights bf16 bits. + +def private bf16_bits_of(v : float) : uint16 { + let bits = unsafe(reinterpret(v)) + return uint16((bits + 0x7FFFu + ((bits >> 16u) & 1u)) >> 16u) +} + +def bench_bf16_batch_tile() : string { + let vs <- bf16_batch_tile_template_variants() + let D = 4l + var inscope w16 : array + var inscope x, y : array + w16 |> resize(4l * N) + var xp = aligned_f32(x, 4l * N) + var yp = aligned_f32(y, 4l * D) + unsafe { + for (i in range64(4l * N)) { + w16[i] = bf16_bits_of(sin(float(i) * 0.1f)) + xp[i] = cos(float(i) * 0.07f) + } + } + let wp = unsafe(addr(w16[0])) + var sink = 0.0f + var inscope race <- race_variants(vs, $(i : int) : bool { + let f = vs[i]._1 + invoke(f, yp, wp, xp, N, D) + var maxrel = 0.0lf + for (r in range64(4l)) { + for (tk in range64(4l)) { + var want = 0.0lf + for (j in range64(N)) { + want += double(unsafe(reinterpret(uint(w16[r * N + j]) << 16u))) * double(unsafe(xp[tk * N + j])) + } + let rel = rel_err(double(unsafe(yp[tk * D + r])), want) + maxrel = worst_rel(maxrel, rel) + } + } + return maxrel <= TOL + }, $(_i : int) {}, $(i : int) { + let f = vs[i]._1 + for (_rep in range(reps_of(i))) { + invoke(f, yp, wp, xp, N, D) + } + }, $ { + sink += y[0] + }) + tune_detail(" checksum {sink}\n") + return report("bf16_batch_tile", race) +} + // ===== gelu_erf: in place over N, gated f64-relative against the erf series' GELU on a [-4, 4] spread. def private erf_series(x : double) : double { @@ -2031,7 +2083,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; ("cvt_q8kv_to_f32", @@bench_cvt_q8kv_to_f32), ("quantize_q8kv_row", @@bench_quantize_q8kv_row), ("dot_mx4q8", @@bench_dot_mx4q8), ("quantize_q8_0_into_ptr", @@bench_quantize), ("quantize_q8_0_bs_into_ptr", @@bench_quantize_bs), ("rope_scaled_neox_tab", @@bench_rope_tab), - ("gemm_f32_uk_4x16", @@bench_gemm_tile), ("f32_batch_tile", @@bench_f32_batch_tile), ("gelu_erf", @@bench_gelu_erf), + ("gemm_f32_uk_4x16", @@bench_gemm_tile), ("f32_batch_tile", @@bench_f32_batch_tile), ("bf16_batch_tile", @@bench_bf16_batch_tile), ("gelu_erf", @@bench_gelu_erf), ("dot_q8q8_laneq4x4", @@bench_laneq4x4) ] let partialMint = !empty(tune_only_filter()) diff --git a/modules/dasLLAMA/tests/test_matmul_batch.das b/modules/dasLLAMA/tests/test_matmul_batch.das index 226cecda26..39b9253c98 100644 --- a/modules/dasLLAMA/tests/test_matmul_batch.das +++ b/modules/dasLLAMA/tests/test_matmul_batch.das @@ -190,6 +190,36 @@ def test_matmul_batch_nest_arm(t : T?) { } } +//! the weight as bf16 bits (round to nearest even), and the oracle over the widened value +def private bf16_bits_of(v : float) : uint16 { + let bits = unsafe(reinterpret(v)) + return uint16((bits + 0x7FFFu + ((bits >> 16u) & 1u)) >> 16u) +} + +//! the bf16 plane through the same walk: Y against the plain oracle over the widened weights within +//! the tile's rounding, a NaN sentinel under every output, the poisoned expectation the control +def check_bf16(t : T?; n, d, ntok : int64) { + let wf <- fillf(n * d) + let x <- fillf(ntok * n) + var w16 : array + w16 |> resize(n * d) + var wwide : array + wwide |> resize(n * d) + for (k in range64(n * d)) { + w16[k] = bf16_bits_of(wf[k]) + wwide[k] = unsafe(reinterpret(uint(w16[k]) << 16u)) + } + var want <- oracle_fp32(wwide, x, n, d, ntok) + var y <- nan_fill(ntok * d) + with_shipped_knobs() { + matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) + } + t |> success(all_finite(y), "bf16 batch {n}x{d} ntok={ntok}: every output finite") + t |> equal(mismatch_rel("bf16 batch {n}x{d} ntok={ntok}", y, want, F32_TILE_REL), 0, "bf16 batch {n}x{d} ntok={ntok}: elements past {F32_TILE_REL} relative of the plain oracle over the widened weights") + want[(ntok * d) / 2l] += 1.0e-3 * max(abs(want[(ntok * d) / 2l]), 1.0) + t |> success(mismatch_rel("bf16 batch {n}x{d} ntok={ntok} poisoned", y, want, F32_TILE_REL) > 0, "control: the poisoned element must miss the {F32_TILE_REL} bar") +} + // Q8·Q8: same, with weights quantized once and the activation quantized per token. def check_q8(t : T?; n, d, ntok : int64) { let nb = n / 32l @@ -252,6 +282,15 @@ def test_matmul_batch(t : T?) { t |> run("fp32 tiles with row and token tails 100x13 ntok=7") @(t : T?) { check_fp32(t, 100l, 13l, 7l) } + t |> run("bf16 plane 1152x480 ntok=300") @(t : T?) { + check_bf16(t, 1152l, 480l, 300l) + } + t |> run("bf16 plane, row and token tails 100x13 ntok=7") @(t : T?) { + check_bf16(t, 100l, 13l, 7l) + } + t |> run("bf16 plane ntok=1, the gemv shaper") @(t : T?) { + check_bf16(t, 96l, 7l, 1l) + } t |> run("fp32 threaded 1152x480 ntok=300") @(t : T?) { check_fp32(t, 1152l, 480l, 300l) } From 31909fefa28d414fca22ee4c35bab30bb8a47efe Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 19:22:59 -0700 Subject: [PATCH 05/18] dasLLAMA: the qwen3a mel splits its DFT GEMM and mel fold over frames log_mel_qwen3a ran its [frames x 402] DFT GEMM and the double-accumulated mel fold on one lane - 400 ms of the Xeon's Qwen3-ASR encode on gb1. Both loops now split over frames, the GEMM on whole 4-row tiles so gemm_f32's scalar strip sees only the global tail; every frame's arithmetic is the one-lane run's, so the mel lands the same bits at any lane count (6.5x on the M5 over a 60 s clip). Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/dasllama/dasllama_qwen3a.das | 78 ++++++++++++------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_qwen3a.das b/modules/dasLLAMA/dasllama/dasllama_qwen3a.das index 1c16aa7b93..220b486f07 100644 --- a/modules/dasLLAMA/dasllama/dasllama_qwen3a.das +++ b/modules/dasLLAMA/dasllama/dasllama_qwen3a.das @@ -16,6 +16,7 @@ require dasllama/dasllama_env require dasllama/dasllama_plane public // PlaneF/PlaneI8 — the borrowed weight planes; re-exported for deleters require dasllama/dasllama_asr_types public // TranscribeSegment/AsrCaps — the shared ASR floor require dasllama/dasllama_math +require dasllama/dasllama_par // maybe_parallel_for - the mel's frame split require dasllama/dasllama_gguf require dasllama/dasllama_audio require dasllama/dasllama_tower @@ -418,36 +419,57 @@ def log_mel_qwen3a(samples : array | #; n_mel : int64) : WhisperMel { if (!mel_on_gpu) { var inscope reim : array reim |> reserve_resize(int(m.n_len * n_bins * 2l)) - gemm_f32(unsafe(addr(reim[0])), unsafe(addr(xw[0])), unsafe(addr(twiddles[0])), - m.n_len, n_fft, n_bins * 2l) - - // power -> mel: float products in groups of 4 summed in float, groups accumulated in - // double (reference-exact rounding), floor 2^-24, log10 - let mel_floor = 5.960464477539063e-08lf - let log10_e = 0.43429448190325176lf - var inscope power : array - power |> resize(int(n_bins)) - for (i in range64(m.n_len)) { - for (k in range64(n_bins)) { - let re = reim[i * n_bins * 2l + k * 2l] - let im = reim[i * n_bins * 2l + k * 2l + 1l] - power[k] = re * re + im * im - } - for (j in range64(n_mel)) { - var sum = 0.0lf - var k = 0l - while (k + 3l < n_bins) { - sum += double(power[k] * filters[j * n_bins + k] - + power[k + 1l] * filters[j * n_bins + k + 1l] - + power[k + 2l] * filters[j * n_bins + k + 2l] - + power[k + 3l] * filters[j * n_bins + k + 3l]) - k += 4l + let n_len = m.n_len + unsafe { + // every frame is its own row of the DFT GEMM and its own mel fold, so both split over + // frames with the per-element arithmetic - and the bits - of the one-lane run + var reimp = addr(reim[0]) + let xwp = addr(xw[0]) + let twp = addr(twiddles[0]) + let fltp = addr(filters[0]) + var md = addr(m.data[0]) + // whole 4-row tiles a chunk: gemm_f32 runs its rows past the last tile through the scalar + // strip, so a chunk edge off the tile lattice would round those rows apart from the one-lane run + let n_tiles = (n_len + 3l) / 4l + maybe_parallel_for(0, int(n_tiles), lanes_for_work(n_len * n_fft * n_bins * 2l, 0)) $(tb, te) { + let r0 = int64(tb) * 4l + let r1 = min(int64(te) * 4l, n_len) + unsafe { + gemm_f32(reimp + r0 * n_bins * 2l, xwp + r0 * n_fft, twp, r1 - r0, n_fft, n_bins * 2l) } - while (k < n_bins) { - sum += double(power[k] * filters[j * n_bins + k]) - k++ + } + // power -> mel: float products in groups of 4 summed in float, groups accumulated in + // double (reference-exact rounding), floor 2^-24, log10 + let mel_floor = 5.960464477539063e-08lf + let log10_e = 0.43429448190325176lf + maybe_parallel_for(0, int(n_len), lanes_for_work(n_len * n_mel * n_bins, 0)) $(rb, re) { + var inscope power : array + power |> resize(int(n_bins)) + unsafe { + for (i in range64(int64(rb), int64(re))) { + for (k in range64(n_bins)) { + let rv = reimp[i * n_bins * 2l + k * 2l] + let iv = reimp[i * n_bins * 2l + k * 2l + 1l] + power[k] = rv * rv + iv * iv + } + for (j in range64(n_mel)) { + var sum = 0.0lf + var k = 0l + while (k + 3l < n_bins) { + sum += double(power[k] * fltp[j * n_bins + k] + + power[k + 1l] * fltp[j * n_bins + k + 1l] + + power[k + 2l] * fltp[j * n_bins + k + 2l] + + power[k + 3l] * fltp[j * n_bins + k + 3l]) + k += 4l + } + while (k < n_bins) { + sum += double(power[k] * fltp[j * n_bins + k]) + k++ + } + md[j * n_len + i] = float(log(max(sum, mel_floor)) * log10_e) + } + } } - m.data[j * m.n_len + i] = float(log(max(sum, mel_floor)) * log10_e) } } } From 55a8b8abf2d1c0ad952c0c651546f01b0eb13cc6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 19:39:40 -0700 Subject: [PATCH 06/18] dasLLAMA: the bf16-weight batch GEMM as a [tune] family with an AMX tile bf16w_tile_gen packs sixteen rows of a native bf16 plane as the TMUL B layout (k-pair rows, bf16w_panel_build), rounds the f32 activations to bf16 once per call (the reference encoders' arithmetic) and runs the quant families' AMX bf16 tile emitter over them; every tail - a short group, the token tail, the row tail - takes the widen walk over the same rounded activations, so one call's outputs share their activation bits. matmul_bf16_batch reaches it through a seam dasllama_math_gen registers; where the family declines (no sidecar entry, no AMX, n off the 32 lattice) the widen walk serves as before. On the Granite Rapids box the stamped tile reads 3886 GFLOP/s at the E4B projection's pp512 shape against the f32 walk's 775, 2931 against 946 at the Qwen3-ASR conv's. The GEMM test's bf16 cells round their oracle's activations where the family serves and hold every tail shape. Co-Authored-By: Claude Fable 5.1 --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 20 +++ modules/dasLLAMA/dasllama/dasllama_math.das | 11 ++ .../dasLLAMA/dasllama/dasllama_math_gen.das | 138 ++++++++++++++++++ modules/dasLLAMA/tests/test_matmul_batch.das | 15 +- 4 files changed, 182 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index c7802e2319..73d0783a72 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -3063,6 +3063,23 @@ def private tokstep_gen(var gc : LlvmCodeCtx) : bool { def private tileform_gen(var gc : LlvmCodeCtx) : bool => int_companion(gc) $(p) => dot_kind(p.dotPrim) == DOT_AMX_BF16 ? p.nrsplit : 0 +//! The bf16-weight tile family: a native bf16 plane packed as the AMX B layout (k-pair rows of +//! sixteen rows, `bf16w_panel_build`) against a bf16 activation plane; the amx_bf16 perm is the +//! TMUL tile over them - the same emitter as the quant families' bf16 leg, the panel already bf16 +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#amx-bf16-tile")] +def private bf16w_tile_gen(var gc : LlvmCodeCtx) : bool { + if (!is_tile_signature(gc.fn)) return false + let p = parse_perm(gc) + if (perm_declines(gc, p) || dot_kind(p.dotPrim) != DOT_AMX_BF16) return false + return emit_amx_bf16_tile(gc, p) +} + +def private bf16w_tokstep_gen(var gc : LlvmCodeCtx) : bool => + int_companion(gc) $(p) => dot_kind(p.dotPrim) == DOT_AMX_BF16 ? 32 : 4 + +def private bf16w_tileform_gen(var gc : LlvmCodeCtx) : bool => + int_companion(gc) $(p) => dot_kind(p.dotPrim) == DOT_AMX_BF16 ? p.nrsplit : 0 + def private is_witness_signature(fn : Function const?) : bool => nullary_signature(fn, Type.tBool) // The witness: `() : bool` — did the stamped family actually EMIT on this target? Reference @@ -5009,6 +5026,9 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::kq_amx_cfg", @@cfg_gen) register_llvm_code_generator("dasllama_gemm_gen::kq_witness", @@witness_gen) register_llvm_code_generator("dasllama_gemm_gen::kq_tileform", @@tileform_gen) + register_llvm_code_generator("dasllama_gemm_gen::bf16w_tile", @@bf16w_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::bf16w_tokstep", @@bf16w_tokstep_gen) + register_llvm_code_generator("dasllama_gemm_gen::bf16w_tileform", @@bf16w_tileform_gen) register_llvm_code_generator("dasllama_gemm_gen::q8_panel", @@q8_panel_gen) register_llvm_code_generator("dasllama_gemm_gen::q8_panel_s16", @@q8_panel_s16_gen) for (f in type) { diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 7928545482..37fc2a5da3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -902,6 +902,14 @@ def private restore_float_override_defaults { def float_batch_override_active() : bool => g_fp_override_active +//! the stamped bf16-weight tile family's batch walk (dasllama_math_gen registers it): true when it +//! served the call, false where the family declines or the shape is off its lattice +typedef Bf16BatchGenFn = function<(var yp : float?; wp : uint16 const?; xp : float const?; n : int64; d : int64; ntok : int64) : bool> +var private g_bf16_batch_gen : Bf16BatchGenFn +def set_bf16_batch_gen(fn : Bf16BatchGenFn) { + g_bf16_batch_gen = fn +} + // the one body: the float-override seam then the core. Both weight forms below reach it with a // base pointer, so a borrowed plane and an owned blob take the identical route. def private matmul_batch_at(var y : array; wp : float const?; x : array; n, d, ntok : int64) { @@ -965,6 +973,9 @@ def private matmul_bf16_batch_at(var y : array; wp : uint16 const?; x : a return } if (ntok > 1l) { + if (g_bf16_batch_gen != null && invoke(g_bf16_batch_gen, yp, wp, xp, n, d, ntok)) { + return + } matmul_batch_core(yp, wp, xp, n, d, ntok) return } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 77622b4408..84fbb915d0 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -4008,6 +4008,143 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k } } +// ===== the bf16-weight batch family: a native bf16 plane (a tower's or the gemma-E projection's) against +// f32 activations, through the AMX bf16 tile - the activations rounded to bf16 as the reference's are + +//! the vector contract's reference values: no tile form, four tokens a call +def bf16w_tokstep_gen() : int => 4 +def bf16w_tileform_gen() : int => 0 +def bf16w_amx_cfg_gen() : void {} +def bf16w_witness_gen() : bool => false + +[tune_perm(mr = 16), + tune_perm(dot = "amx_bf16", width = 512, mr = 16, kstep = 1, nrsplit = 2, requires = "amx-bf16,avx512bf16,avx512vnni"), + tune_perm(dot = "amx_bf16", width = 512, mr = 16, kstep = 1, nrsplit = 1, requires = "amx-bf16,avx512bf16,avx512vnni"), + tune_companion(fn = "bf16w_tokstep_gen", gen = "dasllama_gemm_gen::bf16w_tokstep"), + tune_companion(fn = "bf16w_tileform_gen", gen = "dasllama_gemm_gen::bf16w_tileform"), + tune_companion(fn = "bf16w_amx_cfg_gen", gen = "dasllama_gemm_gen::kq_amx_cfg"), + tune_companion(fn = "bf16w_witness_gen", gen = "dasllama_gemm_gen::kq_witness"), + tune(gen = "dasllama_gemm_gen::bf16w_tile", fallback = "mr16"), + hint(unsafe_range_check, noalias = yp, noalias = panel, noalias = xb), unused_argument(_sp, _xs, _xbs)] +def bf16w_tile_gen(var yp : float?; panel : uint8 const?; _sp : float const?; xb : int8 const?; _xs : float const?; _xbs : int const?; n, d, g, t0 : int64) : void { + // the reference: group g's sixteen rows by four tokens from t0, every product the widened bf16 pair + unsafe { + let pw = reinterpret(panel) + let xw = reinterpret(xb) + for (c in range64(16l)) { + for (t in range64(4l)) { + var acc = 0.0 + let xrow = xw + (t0 + t) * n + for (kp in range64(n / 2l)) { + let w0 = reinterpret(uint(pw[kp * 32l + c * 2l]) << 16u) + let w1 = reinterpret(uint(pw[kp * 32l + c * 2l + 1l]) << 16u) + acc = mad(w0, reinterpret(uint(xrow[2l * kp]) << 16u), acc) + acc = mad(w1, reinterpret(uint(xrow[2l * kp + 1l]) << 16u), acc) + } + yp[(t0 + t) * d + g * 16l + c] = acc + } + } + } +} + +var private @scratch @exact_size g_bf16w_x : array +var private @scratch @exact_size g_bf16w_xr : array + +//! the bf16 activation plane: `ntok` rows of `n`, each f32 rounded to nearest even, 64-byte aligned; +//! `xr` receives the same rounded values widened back to f32, the tails' operand, so every output of +//! one call multiplies the same activation bits +def xbf16_build_f32(xp : float const?; n, ntok : int64; var xr : float?&) : uint8 const? { + g_bf16w_x |> overwrite_resize(ntok * n + 32l) + g_bf16w_xr |> overwrite_resize(ntok * n) + unsafe { + var hx = bf16_align64(addr(g_bf16w_x[0])) + var rp = addr(g_bf16w_xr[0]) + let jobs = is_job_que_available() ? int(min(ntok, 16l)) : 1 + maybe_parallel_for(0, int(ntok), jobs) $(tb, te) { + unsafe { + for (t in range(tb, te)) { + let xrow = xp + int64(t) * n + var hrow = hx + int64(t) * n + var rrow = rp + int64(t) * n + for (i in range64(n)) { + let h = bf16_bits(xrow[i]) + hrow[i] = h + rrow[i] = reinterpret(uint(h) << 16u) + } + } + } + } + xr = rp + return reinterpret(hx) + } +} + +//! `groups` sixteen-row groups from row g0 * 16 of a bf16 plane as the tile's k-pair rows: group gi's +//! row c holds weights (2kp, 2kp + 1) at halfword gi * 16n + kp * 32 + 2c - one uint32 move a pair +def bf16w_panel_build(wp : uint16 const?; n, g0, groups : int64; var dst : uint16?) { + unsafe { + let n2 = n / 2l + for (gi in range64(groups)) { + var dst32 = reinterpret(dst + gi * 16l * n) + for (c in range64(16l)) { + let src32 = reinterpret(wp + ((g0 + gi) * 16l + c) * n) + for (kp in range64(n2)) { + dst32[kp * 16l + c] = src32[kp] + } + } + } + } +} + +//! the bf16-weight batch GEMM through the stamped tile family, units of `tileform` groups over whole +//! `tokstep` blocks, every tail through the widen walk over the same bf16-rounded activations; false +//! where the family declines or the shape is off its lattice (n % 32, d under 16, ntok under a tile) +[hot_path] +def bf16w_batch(var yp : float?; wp : uint16 const?; xp : float const?; n, d, ntok : int64) : bool { + let form = int64(bf16w_tileform_gen()) + if (form <= 0l || n % 32l != 0l || d < 16l || !bf16w_witness_gen()) return false + let ts = int64(bf16w_tokstep_gen()) + if (ntok < ts) return false + let ng = d / 16l + let nu = int((ng + form - 1l) / form) + let tbs = (ntok / ts) * ts + var xr : float? + let xb = xbf16_build_f32(xp, n, ntok, xr) + let xrc = unsafe(reinterpret(xr)) + var panels = walk_panels(form * 16l * n * 2l) + var pbase = panels.base + let pstride = panels.stride + let chunks = matmul_chunks(int(ng * 16l), get_q8_batch_chunks_per_job(), n * d * ntok) + var myp = yp + maybe_parallel_for_indexed(0, nu, min(chunks, nu)) $(slot, ub, ue) { + bf16w_amx_cfg_gen() + unsafe { + var pp = reinterpret(pbase + int64(slot) * pstride) + for (u in range(ub, ue)) { + let g0 = int64(u) * form + let gn = min(form, ng - g0) + if (gn == form) { + bf16w_panel_build(wp, n, g0, form, pp) + var tk = 0l + while (tk < tbs) { + bf16w_tile_gen(myp, reinterpret(pp), null, reinterpret(xb), null, null, n, d, g0, tk) + tk += ts + } + if (tbs < ntok) { + batch_tile_block(myp, wp, xrc, n, d, g0 * 16l, (g0 + form) * 16l, tbs, ntok, ntok - tbs) + } + } else { + batch_tile_block(myp, wp, xrc, n, d, g0 * 16l, (g0 + gn) * 16l, 0l, ntok, ntok) + } + } + } + } + if (ng * 16l < d) { + batch_tile_block(yp, wp, xrc, n, d, ng * 16l, d, 0l, ntok, ntok) + } + return true +} + //! the bf16 walk's token block: the knob or the floor, whichever is larger, in whole tiles and under no L2 clamp - the walk builds a row group's panel once per block, so a block the clamp cuts pays a second panel and a token tail def bf16_token_block() : int64 { let ts = amx_bf16_tile_tokens() @@ -4385,6 +4522,7 @@ def dasllama_math_gen_register() { set_bake_b32_repack(KqFmt.q4032, @@repack_q4032_bake) set_bake_b32_repack(KqFmt.q5032, @@repack_q5032_bake) set_bake_q8q8_repack(@@repack_q8q8_bake) + set_bf16_batch_gen(@@bf16w_batch) set_bake_mx4_repack(@@repack_mx4_bake) set_bake_kq_repack(@@repack_kq_bake) // needs_repack keeps this out of auto-select (direct callers stay on arm64-sdot). q8_layout diff --git a/modules/dasLLAMA/tests/test_matmul_batch.das b/modules/dasLLAMA/tests/test_matmul_batch.das index 39b9253c98..f84ee92b2f 100644 --- a/modules/dasLLAMA/tests/test_matmul_batch.das +++ b/modules/dasLLAMA/tests/test_matmul_batch.das @@ -5,6 +5,7 @@ options persistent_heap // a dastest root frees what it deletes only on this h require dastest/testing_boost public require dasllama/dasllama_math require dasllama/dasllama_math_default // registers the portable Q8·Q8 backend (the matmul_q8q8 wrappers dispatch to it) +require dasllama/dasllama_math_gen // bf16w_tileform_gen: the bf16 plane's walk rounds its activations where the half-width family serves require dasllama/dasllama_convert require daslib/jobque_boost require _compares // row_fill @@ -197,10 +198,17 @@ def private bf16_bits_of(v : float) : uint16 { } //! the bf16 plane through the same walk: Y against the plain oracle over the widened weights within -//! the tile's rounding, a NaN sentinel under every output, the poisoned expectation the control +//! the tile's rounding, a NaN sentinel under every output, the poisoned expectation the control; +//! where the box stamps the half-width family (`bf16w_tileform_gen() > 0`, every call of ntok past a +//! tile on an n of whole 32s) the oracle's activations are the family's bf16-rounded ones def check_bf16(t : T?; n, d, ntok : int64) { let wf <- fillf(n * d) - let x <- fillf(ntok * n) + var x <- fillf(ntok * n) + if (bf16w_tileform_gen() > 0 && n % 32l == 0l && ntok > 1l && ntok >= int64(bf16w_tokstep_gen())) { + for (k in range64(ntok * n)) { + x[k] = unsafe(reinterpret(uint(bf16_bits_of(x[k])) << 16u)) + } + } var w16 : array w16 |> resize(n * d) var wwide : array @@ -288,6 +296,9 @@ def test_matmul_batch(t : T?) { t |> run("bf16 plane, row and token tails 100x13 ntok=7") @(t : T?) { check_bf16(t, 100l, 13l, 7l) } + t |> run("bf16 plane, a group tail and a token tail 2560x200 ntok=100") @(t : T?) { + check_bf16(t, 2560l, 200l, 100l) + } t |> run("bf16 plane ntok=1, the gemv shaper") @(t : T?) { check_bf16(t, 96l, 7l, 1l) } From a181b198384f523666ed4923b6df5e838130ccd7 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 19:55:43 -0700 Subject: [PATCH 07/18] dasLLAMA: the generator probe gates and races the bf16-weight family bf16w_tile_gen joins the probe's two rails: test mode holds every variant - the reference row through the widen walk, a tile-form row through its panel and tokstep walk - to a double oracle over the bf16-rounded activations on a shape with every tail and the Qwen3-ASR conv's shape; tune mode times the tile-form rows and the widen walk on the gemma-E projection's pp512 shape and writes the fastest as the family's sidecar entry (the sticky margin applies). Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/harness/gen_tune_probe.das | 220 +++++++++++++++++++- 1 file changed, 218 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 59e0ef2668..03cf04a42e 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -926,6 +926,203 @@ def b32_tune_family(fmt : KqFmt; var rows_out : table) : str return best } +// ===== the bf16-weight tile family (bf16w_tile_gen): a native bf16 plane against f32 activations ===== +// Its walk is bf16w_batch's, one lane: a panel of `form` groups a unit, `tokstep` tiles over the +// whole token blocks, the widen walk over the same rounded activations for every tail. The oracle +// is a double dot over the bf16-rounded activations and the widened weights, so the stamped tile +// and the reference row answer to one truth within close_tol. + +typedef Bf16wTileFn = function<(var yp : float?; panel : uint8 const?; sp : float const?; xb : int8 const?; xs : float const?; xbs : int const?; n : int64; d : int64; g : int64; t0 : int64) : void> + +struct Bf16wFixture { + n : int64 + d : int64 + ntok : int64 + w16 : array + x : array + yref : array +} + +def private bf16_round(v : float) : float { + let bits = unsafe(reinterpret(v)) + return unsafe(reinterpret(((bits + 0x7FFFu + ((bits >> 16u) & 1u)) >> 16u) << 16u)) +} + +def build_bf16w_fixture(n, d, ntok : int64) : Bf16wFixture { + var fx = Bf16wFixture(n = n, d = d, ntok = ntok) + fx.w16 |> resize(n * d) + fx.x |> resize(ntok * n) + fx.yref |> resize(ntok * d) + for (i in range64(n * d)) { + let w = bf16_round(sin(float(i) * 0.013) * 0.7) + fx.w16[i] = uint16(unsafe(reinterpret(w)) >> 16u) + } + for (i in range64(ntok * n)) { + fx.x[i] = cos(float(i) * 0.0071) * 0.9 + } + for (t in range64(ntok)) { + for (r in range64(d)) { + var acc = 0.0lf + for (k in range64(n)) { + acc += double(bf16_round(fx.x[t * n + k])) * double(unsafe(reinterpret(uint(fx.w16[r * n + k]) << 16u))) + } + fx.yref[t * d + r] = float(acc) + } + } + return <- fx +} + +//! one variant over the fixture as bf16w_batch walks it, one lane; form 0 (the reference row) walks one group at four tokens +def run_bf16w_variant(tile : Bf16wTileFn; fx : Bf16wFixture; form, ts : int64; var y : array) { + let groups = max(form, 1l) + let step = form > 0l ? ts : 4l + let ng = fx.d / 16l + let tbs = (fx.ntok / step) * step + var panel : array + panel |> resize(groups * 16l * fx.n) + unsafe { + let wp = addr(fx.w16[0]) + var yp = addr(y[0]) + var xr : float? + let xb = xbf16_build_f32(addr(fx.x[0]), fx.n, fx.ntok, xr) + let xrc = reinterpret(xr) + var pp = addr(panel[0]) + var g0 = 0l + while (g0 < ng) { + let gn = min(groups, ng - g0) + if (gn == groups) { + bf16w_panel_build(wp, fx.n, g0, groups, pp) + var tk = 0l + while (tk < tbs) { + invoke(tile, yp, reinterpret(pp), null, reinterpret(xb), null, null, fx.n, fx.d, g0, tk) + tk += step + } + if (tbs < fx.ntok) { + batch_tile_block(yp, wp, xrc, fx.n, fx.d, g0 * 16l, (g0 + groups) * 16l, tbs, fx.ntok, fx.ntok - tbs) + } + } else { + batch_tile_block(yp, wp, xrc, fx.n, fx.d, g0 * 16l, (g0 + gn) * 16l, 0l, fx.ntok, fx.ntok) + } + g0 += groups + } + if (ng * 16l < fx.d) { + batch_tile_block(yp, wp, xrc, fx.n, fx.d, ng * 16l, fx.d, 0l, fx.ntok, fx.ntok) + } + } +} + +def close_to_bf16w_ref(fx : Bf16wFixture; y : array; var maxdiff : float&) : bool { + var ok = true + for (i in range64(fx.ntok * fx.d)) { + let diff = abs(y[i] - fx.yref[i]) + maxdiff = max(maxdiff, diff) + ok = ok && diff <= close_tol(fx.yref[i]) + } + return ok +} + +def bf16w_test_family(fxs : array) : bool { + var tvs <- bf16w_tile_gen_variants() + var forms <- forms_of(bf16w_tileform_gen_variants()) + var steps <- forms_of(bf16w_tokstep_gen_variants()) + var wvs <- bf16w_witness_gen_variants() + var cvs <- bf16w_amx_cfg_gen_variants() + var allok = true + for (v in tvs) { + let form = int64(forms?[v._0] ?? 0) + let ts = int64(steps?[v._0] ?? 4) + if (form > 0l) { + arm_of(wvs, cvs, v._0) + } + var ok = true + var maxdiff = 0.0 + for (fx in fxs) { + var inscope y : array + y |> resize(fx.ntok * fx.d) + run_bf16w_variant(v._1, fx, form, ts, y) + let fxok = close_to_bf16w_ref(fx, y, maxdiff) + if (!fxok) { + dump_mismatch("bf16w {v._0} n={fx.n}", y, fx.yref, int(fx.d)) $(i : int) => close_tol(fx.yref[i]) + } + ok = ok && fxok + } + print("bf16w {v._0} (form={form}, tokstep={ts}): {ok ? "ok" : "MISMATCH"} (maxdiff {maxdiff})\n") + allok = allok && ok + } + delete cvs + delete wvs + delete steps + delete forms + delete tvs + return allok +} + +//! the bf16-weight family's tune pass: every variant gated on the fixture, the tile-form variants and the widen walk +//! (the `mr16` row's production route) timed on the gemma-E projection's pp512 shape, the fastest the entry +def bf16w_tune_family(var rows_out : table) : string { + let entry = "bf16w_tile_gen" + var fx <- build_bf16w_fixture(2560l, 1024l, 512l) + var tvs <- bf16w_tile_gen_variants() + var forms <- forms_of(bf16w_tileform_gen_variants()) + var steps <- forms_of(bf16w_tokstep_gen_variants()) + var wvs <- bf16w_witness_gen_variants() + var cvs <- bf16w_amx_cfg_gen_variants() + var names : array + var times : array + var best = "" + var bestDt = 1.0e30lf + var y : array + y |> resize(fx.ntok * fx.d) + for (v in tvs) { + let form = int64(forms?[v._0] ?? 0) + let ts = int64(steps?[v._0] ?? 4) + if (form > 0l) { + arm_of(wvs, cvs, v._0) + } + var maxdiff = 0.0 + run_bf16w_variant(v._1, fx, form, ts, y) + if (!close_to_bf16w_ref(fx, y, maxdiff)) { + print("bf16w {v._0}: MISMATCH (maxdiff {maxdiff}) - aborting tune\n") + best = "" + break + } + var dt = 1.0e30lf + for (_rep in range(3)) { + let t0 = ref_time_ticks() + if (form > 0l) { + run_bf16w_variant(v._1, fx, form, ts, y) + } else { + unsafe { // the reference row serves through the widen walk, not its own body + batch_tile_block(addr(y[0]), addr(fx.w16[0]), addr(fx.x[0]), fx.n, fx.d, 0l, fx.d, 0l, fx.ntok, fx.ntok) + } + } + dt = min(dt, double(get_time_usec(t0))) + } + let gf = 2.0lf * double(fx.n * fx.d * fx.ntok) / dt / 1000.0lf + rows_out[v._0] = JV((tile_us = dt)) + names |> push(v._0) + times |> push(dt) + tune_detail("bf16w {v._0} (form={form}, tokstep={ts}): ok (maxdiff {maxdiff}), {dt} us ({gf:0.1f} GFLOP/s)\n") + if (dt < bestDt) { + bestDt = dt + best = v._0 + } + } + if (!empty(best)) { + best = names[gen_sticky(entry, names, times, find_index(names, best))] + } + delete names + delete times + delete y + delete cvs + delete wvs + delete steps + delete forms + delete tvs + delete fx + return best +} + def kq_test_family(fmt : int64; kfxs : array) : bool { var regs <- kq_regs(fmt) var gtab <- by_suffix(regs.gemv) @@ -1306,6 +1503,10 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs } delete kids + // the bf16-weight family: every tail in the first shape, the conv front's in the second + var inscope bfxs <- [ <- build_bf16w_fixture(2560l, 200l, 100l), <- build_bf16w_fixture(1152l, 480l, 300l)] + allok = bf16w_test_family(bfxs) && allok + delete bfxs delete wtab delete mtab delete gtab @@ -1720,11 +1921,12 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family var kids <- kq_family_ids() var kqSel <- [for (fmt in kids); fmt; where tune_family_selected(kq_tile_entry(fmt))] let wantQ8 = tune_family_selected("q8q8_tile_gen") + let wantBf16w = tune_family_selected("bf16w_tile_gen") let b32All = fixed_array(KqFmt.q51, KqFmt.iq4nl32, KqFmt.q4032, KqFmt.q5032) var b32Sel <- [for (f in b32All); f; where tune_family_selected(b32_tile_entry(f))] - let familyTotal = 1 + length(kids) + length(b32All) // q8q8, the kq families, the per-32 families + let familyTotal = 2 + length(kids) + length(b32All) // q8q8, the kq families, the per-32 families, the bf16-weight family delete kids - let planTotal = (wantQ8 ? 2 : 0) + length(kqSel) + length(b32Sel) + let planTotal = (wantQ8 ? 2 : 0) + length(kqSel) + length(b32Sel) + (wantBf16w ? 1 : 0) if (planTotal == 0) { print("tune-only '{tune_only_filter()}': no generator family matches - half skipped, sidecar untouched\n") delete kqSel @@ -1805,6 +2007,20 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("{entry} winner: {empty(qw) ? "reference (gate-failure fallback)" : qw}\n") } delete b32Sel + if (wantBf16w) { + let entry = "bf16w_tile_gen" + interrupt_gate(entry) + let bf_t0 = ref_time_ticks() + tune_progress_kernel_begin(entry, ROUNDS) + var brows : table + let bw = bf16w_tune_family(brows) + tune_detail("TUNE_GEN_TIME bf16w family {get_time_usec(bf_t0) / 1000} ms\n") + tune_progress_kernel_end(entry, bw, empty(bw) ? "rejected" : "beats") + // a gate failure queues the base row - its production route is the widen walk - so the scope completes + pending[entry] = empty(bw) ? "mr16" : bw + g_rows[entry] = JV(brows) + print("{entry} winner: {empty(bw) ? "mr16 (gate-failure fallback)" : bw}\n") + } var ok = true if (wantQ8) { interrupt_gate("confirm_e2e_prefill") From 156b4a456c1dd8b4eb52e737c8fac1cb156da71b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 19:59:42 -0700 Subject: [PATCH 08/18] dasLLAMA: the bf16-weight family's section and the probe's envelope bar ARCHITECTURE_CPU_KERNELS.md gains the family's section, cited from bf16w_batch and routed from ARCHITECTURE.md; tests/CLAUDE.md carries test_matmul_batch's entry. The probe holds a tile-form row to the bf16 envelope of the largest output, as the quant families' bf16 rows are held: the TMUL sums its pairs in another order than the double oracle, and the fixture's small outputs sat under close_tol's absolute floor on the Granite Rapids box. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/ARCHITECTURE.md | 3 ++- modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md | 17 +++++++++++++++++ modules/dasLLAMA/dasllama/dasllama_math_gen.das | 2 +- modules/dasLLAMA/harness/gen_tune_probe.das | 13 ++++++++----- modules/dasLLAMA/tests/CLAUDE.md | 8 ++++++++ 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index d89c396c33..e39240a36c 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -222,7 +222,8 @@ Each companion's sections are anchored by topic; a citation spells `.md# repack is made of, the bf16 tile that keeps its sum in the matrix unit, the JIT partition a hot leaf instantiates in, and the - fp32 batch tile's lattice (`ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice`); the q8 chain's + fp32 batch tile's lattice (`ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice`) and the bf16 weight + plane's matrix-unit family (`ARCHITECTURE_CPU_KERNELS.md#bf16-weight-tile-family`); the q8 chain's activation forms against the reference's (`ARCHITECTURE_CPU_KERNELS.md#q8-activation-forms`). diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md index ebef0281ce..a189fa9310 100644 --- a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -207,3 +207,20 @@ logits (`mm_fblob_rows`), whose top-k over tile-rounded logits parts from decode routes the token to another expert than decode would (`test_parity.das` holds the 26B-A4B's Q4_0 file to its pinned tokens on the CPU backend; `test_matmul_batch.das` holds the entry to the per-token matmul bit for bit at the router's shape). + +### A bf16 weight plane multiplies as the reference does, through the matrix unit {#bf16-weight-tile-family} + +The reference encoders multiply a bf16 plane - a tower's GEMMs, the gemma-E per-layer projection - +as bf16 times bf16 with the activations rounded to bf16, and so does `bf16w_tile_gen`: `bf16w_batch` +rounds the f32 activations to bf16 once per call (`xbf16_build_f32`, the rounded values kept widened +too), packs sixteen rows of the plane into the TMUL B layout the quant families' bf16 tile already +reads (`bf16w_panel_build`: k-pair rows of sixteen columns, one 32-bit move a pair, since the plane +holds the pairs the tile wants), and runs the same `amx_bf16` emitter over the panel and the +activation plane. A unit is `tileform` groups of sixteen rows over whole `tokstep` blocks; every +tail - a unit short of its groups, the tokens past the last block, the rows past the last group - +takes the widen tile walk over the same rounded activations, so one call's outputs share their +activation bits whatever lattice computed them. The family declines where no sidecar names a perm, +where the box has no AMX bf16, or where n is off the 32 lattice, and the widen walk +(`matmul_batch_core` over the halfword plane) serves instead; the tile's sums come out of the matrix +unit in another order than a sequential dot, so a cell holds the family to the bf16 envelope of the +largest output, not to the widen walk's bits. diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 84fbb915d0..b578adb5a6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -4099,7 +4099,7 @@ def bf16w_panel_build(wp : uint16 const?; n, g0, groups : int64; var dst : uint1 //! the bf16-weight batch GEMM through the stamped tile family, units of `tileform` groups over whole //! `tokstep` blocks, every tail through the widen walk over the same bf16-rounded activations; false //! where the family declines or the shape is off its lattice (n % 32, d under 16, ntok under a tile) -[hot_path] +[hot_path, arch(at = "../ARCHITECTURE_CPU_KERNELS.md#bf16-weight-tile-family")] def bf16w_batch(var yp : float?; wp : uint16 const?; xp : float const?; n, d, ntok : int64) : bool { let form = int64(bf16w_tileform_gen()) if (form <= 0l || n % 32l != 0l || d < 16l || !bf16w_witness_gen()) return false diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 03cf04a42e..4a2faf033e 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1011,12 +1011,15 @@ def run_bf16w_variant(tile : Bf16wTileFn; fx : Bf16wFixture; form, ts : int64; v } } -def close_to_bf16w_ref(fx : Bf16wFixture; y : array; var maxdiff : float&) : bool { +//! `y` against the oracle at close_tol, widened to the bf16 envelope on a tile form: the TMUL sums its +//! pairs in another order than the double oracle, a few f32 ulps of the largest output over 2560 terms +def close_to_bf16w_ref(fx : Bf16wFixture; y : array; form : int64; var maxdiff : float&) : bool { + let envelope = bf16_envelope(fx.yref, length(y), form) var ok = true for (i in range64(fx.ntok * fx.d)) { let diff = abs(y[i] - fx.yref[i]) maxdiff = max(maxdiff, diff) - ok = ok && diff <= close_tol(fx.yref[i]) + ok = ok && diff <= max(close_tol(fx.yref[i]), envelope) } return ok } @@ -1040,9 +1043,9 @@ def bf16w_test_family(fxs : array) : bool { var inscope y : array y |> resize(fx.ntok * fx.d) run_bf16w_variant(v._1, fx, form, ts, y) - let fxok = close_to_bf16w_ref(fx, y, maxdiff) + let fxok = close_to_bf16w_ref(fx, y, form, maxdiff) if (!fxok) { - dump_mismatch("bf16w {v._0} n={fx.n}", y, fx.yref, int(fx.d)) $(i : int) => close_tol(fx.yref[i]) + dump_mismatch("bf16w {v._0} n={fx.n}", y, fx.yref, int(fx.d)) $(i : int) => max(close_tol(fx.yref[i]), bf16_envelope(fx.yref, length(y), form)) } ok = ok && fxok } @@ -1081,7 +1084,7 @@ def bf16w_tune_family(var rows_out : table) : string { } var maxdiff = 0.0 run_bf16w_variant(v._1, fx, form, ts, y) - if (!close_to_bf16w_ref(fx, y, maxdiff)) { + if (!close_to_bf16w_ref(fx, y, form, maxdiff)) { print("bf16w {v._0}: MISMATCH (maxdiff {maxdiff}) - aborting tune\n") best = "" break diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index 8c99f1cb2c..d341b5fd42 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -3701,6 +3701,14 @@ explains). each against an in-test reference; `rel_pos_table` against an fp64 sin/cos reference, its relative-position-0 row exactly (0, 1) pairs, the table of one frame more as the control that must miss the bar. +`test_matmul_batch.das` - model-free: the batched GEMMs against a plain double oracle under a NaN +sentinel - the fp32 tile walk within `F32_TILE_REL` of the oracle at every tail shape with an +`all_finite` assert and a poisoned expectation, the threaded walk bit for bit the one-lane walk (its +control a moved element), the row nest under `set_f32_batch_tile_min_n` bit for bit the per-token +`matmul` (the tile at the shipped knob rounding apart is the control), the bf16 plane through the +same walk at every tail shape and the GEMV shaper, its oracle over the bf16-rounded activations +where the box stamps `bf16w_tile_gen` (every walk pinned to the shipped knobs through +`with_shipped_knobs`), and the Q8 path bit for bit. `test_tower_asr_kernels.das` - model-free: the public tower / audio / TTS kernels with no CPU unit of their own, each against an in-test fp64 reference over a procedural seeded-LCG fixture and each bar carrying its own must-EXCEED poison arm - the padded-width GEMM wrappers From 2fcc7c08008262cdf90794045d2496c94376e0fa Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 20:16:18 -0700 Subject: [PATCH 09/18] dasLLAMA: a flash-decode unit is one slice over every kv head The sliced decode attention dispatched a unit per (kv head, slice), so the eight kv heads' units read each K/V row's 2 KB in eight 256-byte pieces at eight different times. A unit is now one slice over every kv head: each row streams once, whole. The per-(head, slice) partials are computed as before, so the outputs are bit for bit the old ones (the fused-decode cells hold). In a K-pass emulation the order halves the Granite Rapids box's time a layer (220 to 112 us at 3000 keys over 8 lanes) and lifts the M5's a third; the box's full chain did not move (8.7 against 8.9 s over 451 steps), so the chain's wall sits elsewhere - its measured split is in the ledger. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/dasllama/dasllama_blocks.das | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 63baa6cb13..34a9c88bad 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -121,10 +121,10 @@ def private attention_chain_decode(t : Model; var s : Session; l : int64; pos : s.chain_stages |> resize(5) s.chain_stages[0] = int3(0, ngq, n0) s.chain_stages[1] = int3(0, nslots, nslots) - s.chain_stages[2] = int3(0, int(n_kv_heads) * nsl, int(n_kv_heads) * nsl) + s.chain_stages[2] = int3(0, nsl, nsl) s.chain_stages[3] = int3(0, nhu, nhu) s.chain_stages[4] = int3(0, ngrp, n2) - count_parallel_dispatch(min(n0, ngq) + nslots + int(n_kv_heads) * nsl + nhu + min(n2, ngrp)) + count_parallel_dispatch(min(n0, ngq) + nslots + nsl + nhu + min(n2, ngrp)) } else { s.chain_stages |> resize(4) s.chain_stages[0] = int3(0, ngq, n0) @@ -350,15 +350,15 @@ def private attention_chain_decode(t : Model; var s : Session; l : int64; pos : } } } elif (stage == 2) { - // flash-decode partials: unit = (KV-head, slice) — K/V rows stream ONCE, serving all kv_mul q-heads + // flash-decode partials: unit = one slice over EVERY kv head - each K/V row streams once, whole unsafe { for (u in range(jb, je)) { - let kvh = int64(u / nsl) - let sl = int64(u % nsl) - let kvhoff = kvh * head_size - let qh0 = kvh * kv_mul + let sl = int64(u) let j0 = sl * slice_len let j1 = min(j0 + slice_len, cnt) + for (kvh in range64(n_kv_heads)) { + let kvhoff = kvh * head_size + let qh0 = kvh * kv_mul // K pass: each K row read once, dotted against every group q-head kv_slice_scores_d(attp, qp, attqp, attsp, kcl, kdt, runsp, nrun, kstart, j0, j1, kv_dim, kvhoff, head_size, qh0, kv_mul, seq_len, scale) @@ -403,6 +403,7 @@ def private attention_chain_decode(t : Model; var s : Session; l : int64; pos : // V pass: each V row read once, weighted into every group head's partial kv_slice_av_d(flop, attp, vcl, vdt, runsp, nrun, kstart, j0, j1, kv_dim, kvhoff, head_size, qh0, kv_mul, seq_len, int64(nsl), sl) + } } } } elif (stage == 3 && nsl > 1) { From caa61fabc62b6a6a518428b1f80e1cbfa7bd14fd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 20:21:16 -0700 Subject: [PATCH 10/18] dasLLAMA: the changed-cell and re-minted-row rules route to the re-mint Two rules still had a between-release diff write das rows into the record stores, against the release boundary the board carries: a changed board cell now withdraws its rows and writes its new figures into a dated ledger entry beside the old ones, naming the followup row the release re-mint owes, and the re-minted row rule binds the release re-mint. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/REVIEW_MEASUREMENT.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 900062e951..7d8d03f456 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -198,16 +198,20 @@ shipped binary gives. diff runs faster or does less work, where no rig leg runs the changed lines, states in its PR body that no rig leg runs them and names the instrument that does, with its command line.** -**A diff that changes what a board cell loads, runs, counts or times - the model or corpus it -reads, the model components (encoder, decoder, tower) it runs or the backend that serves them, -the tokens or rows it divides the wall by, or the reference build it times (`DEFAULT_REF_SHA` in -`benchmarks/setup_lcpp_ref.das`, or anything else deciding which reference binary or environment -the run measures) - ships before/after rows for each affected cell and corpus, or withdraws the -affected rows and names the withdrawal and its reason in the PR body.** - -**A re-minted row of a changed board cell replaces the old row in `performance/records/.json`, -and a new dated `PERF_LEDGER.md` entry in the same change puts the old figure beside the new one; -a re-minted row that a `held` entry holds deletes that entry in the same change.** +**A diff that changes what a board cell loads, runs, counts or times - the model or corpus it reads, +the model components (encoder, decoder, tower) it runs or the backend that serves them, the tokens +or rows it divides the wall by, or the reference build it times (`DEFAULT_REF_SHA` in +`benchmarks/setup_lcpp_ref.das`, or anything else deciding which reference binary or environment the +run measures) - withdraws the affected rows (the rule below), writes the cell's new figures on the +board boxes it measured into a dated `PERF_LEDGER.md` entry beside the old ones, and names in its PR +body the `followup_general.md` row under which the release re-mint owes the cell's rows; the release +re-mint (`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`) is the diff that writes +them.** + +**A re-minted row of a changed board cell replaces the old row in `performance/records/.json` +in the release re-mint, and a dated `PERF_LEDGER.md` entry in the same change puts the old figure +beside the new one; a re-minted row that a `held` entry holds deletes that entry in the same +change.** **A withdrawn row of a changed board cell is deleted from `performance/records/.json` or gets a `held` entry - a `performance/records/annotations.json` entry carrying the row's `gguf`, `box`, From a936fb369e6a634e49a445167a10bb420bea3a50 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 20:33:28 -0700 Subject: [PATCH 11/18] dasLLAMA: the measurement rules name the re-mint; the CPU ASR races REVIEW_MEASUREMENT.md: the between-release rule and the changed-cell rule write their figures from every board box of the author's and ban every diff but the release re-mint from the record stores; the release re-mint is its own rule (defined at first use), the new-code cell rule adds its cell to the records rig and names the owing row, the fallback rule's trigger is a cell no box of the author's can run or a ledger figure none can measure, the architecture list is the companion glob. ARCHITECTURE_CPU_KERNELS.md: the bf16 family section in short sentences with the full decline list and the probe's envelope bar named; the lattice section covers the bf16 plane the walk now takes. REVIEW.das re-stamps the generator hash - the new keys emit nothing for an existing perm. harness/cpu_asr_race.das races the bf16 plane against the f32 plane, the mel's lane split and the decode K pass's unit orders in-tree, the ledger's instrument for those figures. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/ARCHITECTURE.md | 2 +- modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md | 94 ++++---- modules/dasLLAMA/REVIEW.das | 2 +- modules/dasLLAMA/REVIEW_MEASUREMENT.md | 58 ++--- modules/dasLLAMA/harness/cpu_asr_race.das | 234 +++++++++++++++++++ 5 files changed, 319 insertions(+), 71 deletions(-) create mode 100644 modules/dasLLAMA/harness/cpu_asr_race.das diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index e39240a36c..1b20c72053 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -222,7 +222,7 @@ Each companion's sections are anchored by topic; a citation spells `.md# repack is made of, the bf16 tile that keeps its sum in the matrix unit, the JIT partition a hot leaf instantiates in, and the - fp32 batch tile's lattice (`ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice`) and the bf16 weight + batch tile's lattice (`ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice`) and the bf16 weight plane's matrix-unit family (`ARCHITECTURE_CPU_KERNELS.md#bf16-weight-tile-family`); the q8 chain's activation forms against the reference's (`ARCHITECTURE_CPU_KERNELS.md#q8-activation-forms`). diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md index a189fa9310..42c56dbd73 100644 --- a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -184,43 +184,57 @@ out-norm epilogue. One token of a decode step, one position of a prefill and one B-session batched step all walk those same three pieces, so no caller can round apart from another; what differs is only the lane split over (token, head) and the projections around them. -### The fp32 batch tile walks an absolute lattice {#f32-batch-tile-lattice} - -`matmul_batch_core` computes Y = X*W^T in four-row by four-token tiles (`f32_batch_tile`): each step -along n loads four weights and four inputs and issues sixteen FMAs into sixteen SIMD accumulators, -where one `dot` per output loads two values per FMA. A tile's accumulators and a lone `dot` round -differently, so which kernel computes an output has to be a property of the output alone. The -lattice is the 4x4 grid counted from row 0 and token 0, the same whatever range a chunk covers: the -dispatch unit is a row tile, a token chunk is whole tiles, every token block -(`f32_batch_token_block`: the X rows that fit the smaller of the box's q8 L2 budget and -`g_f32_batch_x_block` bytes, rounded down to whole tiles, or the whole token range when it fits) -starts on the lattice, and only the rows past the last whole row tile of d and the tokens past the -last whole token tile of ntok take `dot` (`batch_dot_rows`). Any row chunking, worker count or token -split then lands the same bits (`test_matmul_batch.das`'s lanes cell holds the threaded walk to the -one-worker walk bit for bit); a batch and the same tokens one at a time do not, and -`test_matmul_batch.das` holds the fp32 path to the per-token dot within `F32_TILE_REL`. -`g_f32_batch_tile_min_n` routes a dot shorter than it to `batch_dot_rows` whole; it ships at 1, so the -tile serves every length, and `harness/f32_batch_race.das` is the lever that races the two constants -and the nest against the tile. `matmul_batch_rows` is the nest as its own entry, under no knob: a -batch whose outputs decide what decode decides on the GEMV's bits takes it - the MoE router's prefill -logits (`mm_fblob_rows`), whose top-k over tile-rounded logits parts from decode's at a near tie and -routes the token to another expert than decode would (`test_parity.das` holds the 26B-A4B's Q4_0 file -to its pinned tokens on the CPU backend; `test_matmul_batch.das` holds the entry to the per-token -matmul bit for bit at the router's shape). - -### A bf16 weight plane multiplies as the reference does, through the matrix unit {#bf16-weight-tile-family} - -The reference encoders multiply a bf16 plane - a tower's GEMMs, the gemma-E per-layer projection - -as bf16 times bf16 with the activations rounded to bf16, and so does `bf16w_tile_gen`: `bf16w_batch` -rounds the f32 activations to bf16 once per call (`xbf16_build_f32`, the rounded values kept widened -too), packs sixteen rows of the plane into the TMUL B layout the quant families' bf16 tile already -reads (`bf16w_panel_build`: k-pair rows of sixteen columns, one 32-bit move a pair, since the plane -holds the pairs the tile wants), and runs the same `amx_bf16` emitter over the panel and the -activation plane. A unit is `tileform` groups of sixteen rows over whole `tokstep` blocks; every -tail - a unit short of its groups, the tokens past the last block, the rows past the last group - -takes the widen tile walk over the same rounded activations, so one call's outputs share their -activation bits whatever lattice computed them. The family declines where no sidecar names a perm, -where the box has no AMX bf16, or where n is off the 32 lattice, and the widen walk -(`matmul_batch_core` over the halfword plane) serves instead; the tile's sums come out of the matrix -unit in another order than a sequential dot, so a cell holds the family to the bf16 envelope of the -largest output, not to the widen walk's bits. +### The batch tile walks an absolute lattice {#f32-batch-tile-lattice} + +`matmul_batch_core` computes Y = X*W^T in four-row by four-token tiles over an f32 plane +(`f32_batch_tile`) or a bf16 plane (`bf16_batch_tile`, each weight widened by the exact high-half +shift as it is read, so the outputs are the f32 walk's over the widened plane): each step along n +loads four weights and four inputs and issues sixteen FMAs into sixteen SIMD accumulators, where one +`dot` per output loads two values per FMA. A tile's accumulators and a lone `dot` round differently, +so which kernel computes an output has to be a property of the output alone. The lattice is the 4x4 +grid counted from row 0 and token 0, the same whatever range a chunk covers: the dispatch unit is a +row tile, a token chunk is whole tiles, every token block (`f32_batch_token_block`: the X rows that +fit the smaller of the box's q8 L2 budget and `g_f32_batch_x_block` bytes, rounded down to whole +tiles, or the whole token range when it fits) starts on the lattice, and only the rows past the last +whole row tile of d and the tokens past the last whole token tile of ntok take a per-output dot +(`batch_dot_rows`: `dot` on an f32 plane, `dot_bf16` on a bf16 one). Any row chunking, worker count +or token split then lands the same bits (`test_matmul_batch.das`'s lanes cell holds the threaded +walk to the one-worker walk bit for bit); a batch and the same tokens one at a time do not, and +`test_matmul_batch.das` holds the fp32 path and the bf16-plane path to the per-token dot over the +widened weights within `F32_TILE_REL`. `g_f32_batch_tile_min_n` routes a dot shorter than it to +`batch_dot_rows` whole; it ships at 1, so the tile serves every length, and +`harness/f32_batch_race.das` is the lever that races the two constants and the nest against the +tile. `matmul_batch_rows` is the nest as its own entry, under no knob: a batch whose outputs decide +what decode decides on the GEMV's bits takes it - the MoE router's prefill logits (`mm_fblob_rows`), +whose top-k over tile-rounded logits parts from decode's at a near tie and routes the token to +another expert than decode would (`test_parity.das` holds the 26B-A4B's Q4_0 file to its pinned +tokens on the CPU backend; `test_matmul_batch.das` holds the entry to the per-token matmul bit for +bit at the router's shape). + +### A bf16 plane multiplies through the matrix unit as the reference does {#bf16-weight-tile-family} + +The reference build multiplies a bf16 weight plane - an encoder tower's GEMMs, gemma-E's per-layer +projection - as bf16 times bf16: it rounds the f32 activations to bf16 first. `bf16w_batch` does the +same through the `bf16w_tile_gen` family where the family serves the call; a declined call +multiplies the f32 activations unrounded. Once per call, `xbf16_build_f32` rounds the activations to +bf16 (nearest even) and also keeps the rounded values widened back to f32. + +For each unit, `bf16w_panel_build` packs the plane's rows into the B-tile layout the quant families' +bf16 tile reads: for each pair of K positions (2k, 2k+1), the pair from each of sixteen rows side by +side. That is one 32-bit move per pair, because the plane already stores each pair adjacent. The +same `amx_bf16` emitter then runs the AMX bf16 tile (`TDPBF16PS`) over the panel and the activation +plane. + +A unit is `tileform` groups of sixteen rows (the perm's `nrsplit`, one or two) over every whole +`tokstep` block of tokens (32 under `amx_bf16`). Three tails go through `batch_tile_block`, the +batch tile walk over the bf16 plane, fed the widened rounded copy: a unit short of `tileform` +groups, the tokens past the last whole block, and the rows past the last whole group. So every +output of one call multiplies the same activation bits, whichever kernel computed it. + +`bf16w_batch` declines the whole call when the selected perm is not `amx_bf16` or the target did not +emit it (`bf16w_tileform_gen` is 0, or `bf16w_witness_gen` is false), when n is not a multiple of +32, when d is under 16, or when ntok is under `tokstep`. A one-token call never reaches it (the GEMV +path serves). A declined call runs `matmul_batch_core` over the f32 activations, unrounded. The +matrix unit adds in a different order from a sequential dot, so the probe +(`harness/gen_tune_probe.das`) holds the family to `bf16_envelope` - the bf16 rounding step at the +largest output - and not to `batch_tile_block`'s bits. diff --git a/modules/dasLLAMA/REVIEW.das b/modules/dasLLAMA/REVIEW.das index d32d08f466..bcdbf144b2 100644 --- a/modules/dasLLAMA/REVIEW.das +++ b/modules/dasLLAMA/REVIEW.das @@ -1799,7 +1799,7 @@ let private GEMM_GEN_FILES = [ "modules/dasLLAMA/dasllama/dasllama_gemm_gen.das", "modules/dasLLAMA/dasllama/dasllama_gemm_schema.das" ] -let private GEMM_GEN_STAMP_HASH = 0x2df9ef21ba7903b5ul +let private GEMM_GEN_STAMP_HASH = 0x151a4437a5d9c67ul [arch(at = "ARCHITECTURE_ENGINE_FORMATS.md#cpu-kernel-tiers")] def private check_gemm_gen_stamp { diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 7d8d03f456..935aeb4ec1 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -1,10 +1,8 @@ # dasLLAMA Measurement Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: `ARCHITECTURE_MEASUREMENT.md`, `ARCHITECTURE_MEASUREMENT_KERNEL_RACE.md`, -`ARCHITECTURE_MEASUREMENT_VK_GEMM_PROBE.md`, `ARCHITECTURE_MEASUREMENT_FAT_START.md`, -`ARCHITECTURE_MEASUREMENT_SERVED.md`. Planned work: `followup_metal.md`, `followup_vulkan.md`, -`followup_general.md`. +docs: `ARCHITECTURE_MEASUREMENT*.md` beside this file. Planned work: `followup_metal.md`, +`followup_vulkan.md`, `followup_general.md`. **A rule here that governs a figure binds only a figure the change writes or changes - in the repo outside `site*/` (repo root), in a commit message, or in its PR body - never one a changed line @@ -167,14 +165,15 @@ entry - that figure goes in the PR that lands the code.** **A diff that lets a served turn, or a media encode a turn takes (an image, a video frame or an audio chunk), reach code no spawned or manual cell's input reaches - a new dispatch arm, a -window-size arm inside a path a cell drives at other sizes, or a changed default route - mints a -cell whose input reaches it, in the same change.** +window-size arm inside a path a cell drives at other sizes, or a changed default route - adds to +`performance/gen_bench_records.das` a cell whose input reaches it, in the same change, and names in +its PR body the `followup_general.md` row under which the release re-mint owes its rows.** -**A change that owes a cell or board row no box of the author's can mint names instead, in the -same change, an artifact that shows the new code ran end to end - a record, a gate output, an -instrument's print - or the profile output that shows the change is faster -(`benchmarks/lcpp_bench.das`'s `forward_profile` rows, or the tier's `DASLLAMA_GPU_PROF=1` token -ledger), with the flags that run took.** +**A change that owes a cell no box of the author's can run, or a ledger figure no box of the +author's can measure, names instead, in the same change, an artifact that shows the new code ran end +to end - a record, a gate output, an instrument's print - or the profile output that shows the +change is faster (`benchmarks/lcpp_bench.das`'s `forward_profile` rows, or the tier's +`DASLLAMA_GPU_PROF=1` token ledger), with the flags that run took.** **A diff that changes `dasllama/dasllama_metal_kernels.das` (the kernel library `metal_decode_init` compiles), `performance/profile_common.das`, or a `dasllama/` module @@ -185,14 +184,20 @@ run printed none.** A fat exe is what `daspkg release --fat ` builds (`DA **A diff whose PR body, or a line it adds or changes, states that a route the tree served before the diff runs faster or does less work, where a rig leg runs the changed lines, writes that leg's walls -on every board box that runs it into a dated `PERF_LEDGER.md` entry naming the exe's commit and the -command, and names in the PR body the `followup_general.md` row under which the release re-mint owes -the board row; the release re-mint -(`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`) is the diff that re-mints the -row in `performance/records/.json` and names it in its PR body.** The board is the module's -committed record of what a release's exe costs to serve; a kernel win that lands in no ledger entry -is invisible to the next regression check, and a row minted between releases shows a speed no -shipped binary gives. +from every board box of the author's that runs it - a board box is one with a +`performance/records/.json` - into a dated `PERF_LEDGER.md` entry naming the exe's commit and +the command, and names in the PR body the `followup_general.md` row under which the release re-mint +- `performance/gen_bench_records.das` run over every board box with a release's exe - owes every +board box's row; no diff but the release re-mint writes those rows into +`performance/records/.json`.** The board is the module's committed record of what a release's +exe costs to serve; a kernel win that lands in no ledger entry is invisible to the next regression +check, and a row minted between releases shows a speed no shipped binary gives +(`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`). + +**The release re-mint re-mints every row a `followup_general.md` row owes it in +`performance/records/.json`, deletes any `held` entry holding one, names each row in its PR +body, and puts each changed cell's last board figure beside the new one in a dated `PERF_LEDGER.md` +entry in the same change.** **A diff whose PR body, or a line it adds or changes, states that a route the tree served before the diff runs faster or does less work, where no rig leg runs the changed lines, states in its PR body @@ -202,16 +207,11 @@ that no rig leg runs them and names the instrument that does, with its command l the model components (encoder, decoder, tower) it runs or the backend that serves them, the tokens or rows it divides the wall by, or the reference build it times (`DEFAULT_REF_SHA` in `benchmarks/setup_lcpp_ref.das`, or anything else deciding which reference binary or environment the -run measures) - withdraws the affected rows (the rule below), writes the cell's new figures on the -board boxes it measured into a dated `PERF_LEDGER.md` entry beside the old ones, and names in its PR -body the `followup_general.md` row under which the release re-mint owes the cell's rows; the release -re-mint (`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`) is the diff that writes -them.** - -**A re-minted row of a changed board cell replaces the old row in `performance/records/.json` -in the release re-mint, and a dated `PERF_LEDGER.md` entry in the same change puts the old figure -beside the new one; a re-minted row that a `held` entry holds deletes that entry in the same -change.** +run measures) - deletes each affected row or gives it a `held` entry, writes the cell's new figures +from every board box of the author's that runs the cell into a dated `PERF_LEDGER.md` entry beside +the old ones, and names in its PR body the `followup_general.md` row under which the release re-mint +owes the cell's rows; no diff but the release re-mint writes those rows into +`performance/records/.json`.** **A withdrawn row of a changed board cell is deleted from `performance/records/.json` or gets a `held` entry - a `performance/records/annotations.json` entry carrying the row's `gguf`, `box`, diff --git a/modules/dasLLAMA/harness/cpu_asr_race.das b/modules/dasLLAMA/harness/cpu_asr_race.das new file mode 100644 index 0000000000..4394511951 --- /dev/null +++ b/modules/dasLLAMA/harness/cpu_asr_race.das @@ -0,0 +1,234 @@ +options gen2 +options _dasllama_internal = true + +// The CPU ASR line's three in-process races, one line a candidate, best of --reps interleaved: +// --arm bf16 the bf16 weight plane against the f32 plane through matmul_batch at the gemma-E +// projection's pp512 shape, the Qwen3-ASR conv's and a vision patch class - the stamped +// bf16w family where the sidecar names a perm, the widen walk elsewhere +// --arm mel log_mel_qwen3a over a 60 s synthetic clip, one lane against the lanes, the two +// images compared element for element +// --arm kpass the decode attention's K pass over a model-sized f16 cache (28 layers x 3000 keys x +// 8 kv heads x 128, two q heads a kv head): the (kv head, slice) unit order against one +// slice over every kv head, one lane and all lanes +// +// DAS_TUNE_MANIFEST= bin/daslang -jit modules/dasLLAMA/harness/cpu_asr_race.das -- [--arm all] [--reps N] + +require dasllama/dasllama_math +require dasllama/dasllama_math_default +require dasllama/dasllama_math_gen +require dasllama/dasllama_kv_codec +require dasllama/dasllama_qwen3a +require dasllama/dasllama_audio +require dasllama/dasllama_par +require daslib/jobque_boost +require daslib/f16_cvt +require daslib/clargs +require math + +[CommandLineArgs] +struct Args { + @clarg_doc = "Which race: bf16 | mel | kpass | all (default: all)" + arm : string = "all" + @clarg_doc = "Timed rounds a candidate, the best kept (default: 5)" + reps : int = 5 + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +def private bf16_bits_of(v : float) : uint16 { + let bits = unsafe(reinterpret(v)) + return uint16((bits + 0x7FFFu + ((bits >> 16u) & 1u)) >> 16u) +} + +def private best_ms(reps : int; blk : block<() : void>) : double { + var best = 1.0e30lf + for (_r in range(reps)) { + let t0 = ref_time_ticks() + invoke(blk) + best = min(best, double(get_time_usec(t0)) / 1000.0lf) + } + return best +} + +def private race_bf16(n, d, ntok : int64; reps : int) { + var w : array + var w16 : array + var x, y : array + w |> reserve_resize(int(n * d)) + w16 |> reserve_resize(int(n * d)) + x |> reserve_resize(int(ntok * n)) + y |> reserve_resize(int(ntok * d)) + for (i in range64(n * d)) { + w[i] = float(i % 13l) * 0.1 - 0.6 + w16[i] = bf16_bits_of(w[i]) + } + for (i in range64(ntok * n)) { + x[i] = float(i % 7l) * 0.2 - 0.7 + } + matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) // warm + matmul_batch(y, w, x, n, d, ntok) + var bf = 1.0e30lf + var ff = 1.0e30lf + for (_r in range(reps)) { + let tb = best_ms(1) { + matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) + } + bf = min(bf, tb) + let tf = best_ms(1) { + matmul_batch(y, w, x, n, d, ntok) + } + ff = min(ff, tf) + } + let fl = 2.0lf * double(n * d * ntok) / 1.0e6lf + print("bf16 n={n} d={d} ntok={ntok}: bf16 plane {bf} ms ({fl / bf} GFLOP/s, {bf16w_tileform_gen() > 0 ? "the stamped family" : "the widen walk"}), f32 plane {ff} ms ({fl / ff} GFLOP/s)\n") +} + +def private race_mel(reps : int) { + var samples : array + samples |> reserve_resize(16000 * 60) + for (i in range(16000 * 60)) { + samples[i] = 0.3 * sin(float(i) * 0.01) + 0.1 * sin(float(i) * 0.37) + 0.05 * float((i * 7919) % 101) / 101.0 + } + set_single_thread_(true) + var inscope a <- log_mel_qwen3a(samples, 128l) + let one = best_ms(reps) { + var inscope m <- log_mel_qwen3a(samples, 128l) + } + set_single_thread_(false) + var inscope b <- log_mel_qwen3a(samples, 128l) + let thr = best_ms(reps) { + var inscope m <- log_mel_qwen3a(samples, 128l) + } + var diff = 0 + for (p, q in a.data, b.data) { + if (p != q) { + diff++ + } + } + print("mel 60 s x 128: one lane {one} ms, the lanes {thr} ms, elements apart {diff} of {length(a.data)}\n") +} + +let KP_L = 28l +let KP_KEYS = 3000l +let KP_HS = 128l +let KP_KVH = 8l +let KP_SEQ = 4096l +let KP_NSL = 32l + +//! the K pass as the chain dispatched it before fcfb1dacd8: a unit a (kv head, slice) +def private kpass_heads(kall : array; q : array; var att : array; lanes : int) { + let kvdim = KP_KVH * KP_HS + unsafe { + let qp = addr(q[0]) + var ap = addr(att[0]) + for (l in range64(KP_L)) { + let kp = addr(kall[l * KP_KEYS * kvdim]) + let slice_len = (KP_KEYS + KP_NSL - 1l) / KP_NSL + maybe_parallel_for(0, int(KP_KVH * KP_NSL), lanes) $(ub, ue) { + unsafe { + for (u in range(ub, ue)) { + let kvh = int64(u) / KP_NSL + let sl = int64(u) % KP_NSL + let j0 = sl * slice_len + let j1 = min(j0 + slice_len, KP_KEYS) + for (j in range64(j0, j1)) { + let krow = kp + j * kvdim + kvh * KP_HS + for (g in range64(2l)) { + ap[(kvh * 2l + g) * KP_SEQ + j] = dot_f16(qp + (kvh * 2l + g) * KP_HS, krow, KP_HS) + } + } + } + } + } + } + } +} + +//! the K pass as the chain dispatches it: a unit one slice over every kv head, each row read once +def private kpass_rows(kall : array; q : array; var att : array; lanes : int) { + let kvdim = KP_KVH * KP_HS + unsafe { + let qp = addr(q[0]) + var ap = addr(att[0]) + for (l in range64(KP_L)) { + let kp = addr(kall[l * KP_KEYS * kvdim]) + let slice_len = (KP_KEYS + KP_NSL - 1l) / KP_NSL + maybe_parallel_for(0, int(KP_NSL), lanes) $(ub, ue) { + unsafe { + for (u in range(ub, ue)) { + let j0 = int64(u) * slice_len + let j1 = min(j0 + slice_len, KP_KEYS) + for (j in range64(j0, j1)) { + let krow = kp + j * kvdim + for (hh in range64(2l * KP_KVH)) { + ap[hh * KP_SEQ + j] = dot_f16(qp + hh * KP_HS, krow + (hh / 2l) * KP_HS, KP_HS) + } + } + } + } + } + } + } +} + +def private race_kpass(reps : int) { + let kvdim = KP_KVH * KP_HS + var kall : array + kall |> reserve_resize(int(KP_L * KP_KEYS * kvdim)) + for (i in range64(KP_L * KP_KEYS * kvdim)) { + kall[i] = uint16(f32_to_f16(0.5 * sin(float(i % 977l) * 0.01))) + } + var q : array + q |> resize(2l * KP_KVH * KP_HS) + for (i in range64(2l * KP_KVH * KP_HS)) { + q[i] = cos(float(i) * 0.03) + } + var att : array + att |> resize(2l * KP_KVH * KP_SEQ) + let mb = double(KP_L * KP_KEYS * kvdim * 2l) / 1.0e6lf + let h1 = best_ms(reps) { + kpass_heads(kall, q, att, 1) + } + let hn = best_ms(reps) { + kpass_heads(kall, q, att, int(KP_KVH * KP_NSL)) + } + let r1 = best_ms(reps) { + kpass_rows(kall, q, att, 1) + } + let rn = best_ms(reps) { + kpass_rows(kall, q, att, int(KP_NSL)) + } + print("kpass {mb} MB of f16 keys a step, (kv head, slice) units: one lane {h1} ms ({mb / h1} GB/s), {KP_KVH * KP_NSL} units {hn} ms ({mb / hn} GB/s, {hn / double(KP_L)} ms a layer)\n") + print("kpass {mb} MB of f16 keys a step, a slice over every kv head: one lane {r1} ms ({mb / r1} GB/s), {KP_NSL} units {rn} ms ({mb / rn} GB/s, {rn / double(KP_L)} ms a layer)\n") +} + +[export] +def main() : int { + var r <- parse_args(type) + if (r |> is_err) { + print("error: {r |> unwrap_err}\n\n") + print_help(get_command_info(type), "cpu_asr_race") + return 1 + } + let cfg <- r |> move_unwrap + if (cfg.help) { + print_help(get_command_info(type), "cpu_asr_race") + return 0 + } + with_dasllama_jobque_() { + if (cfg.arm == "all" || cfg.arm == "bf16") { + race_bf16(2560l, 10752l, 512l, cfg.reps) + race_bf16(4320l, 480l, 800l, cfg.reps) + race_bf16(1152l, 1152l, 256l, cfg.reps) + } + if (cfg.arm == "all" || cfg.arm == "mel") { + race_mel(cfg.reps) + } + if (cfg.arm == "all" || cfg.arm == "kpass") { + race_kpass(cfg.reps) + } + } + return 0 +} From 3732433cc805169399a40361f520a5270ec887f6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 20:57:30 -0700 Subject: [PATCH 12/18] dasLLAMA: a declined perm is no candidate in the bf16 family's race The tune pass timed every row of the grid; on a box without AMX the two amx_bf16 rows decline to the base row's widen walk, tie it, and one of them was written as the Zen4 sidecar's entry. Only tile-form rows and the base row mr16 race now. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/harness/gen_tune_probe.das | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 4a2faf033e..35dcfafd54 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1079,6 +1079,9 @@ def bf16w_tune_family(var rows_out : table) : string { for (v in tvs) { let form = int64(forms?[v._0] ?? 0) let ts = int64(steps?[v._0] ?? 4) + // a tile-form row races; the base row `mr16` stands for the widen walk; a declined perm (form 0 under + // another name) is no candidate - its time would be the base row's and a tie could crown it + continue if (form <= 0l && v._0 != "mr16") if (form > 0l) { arm_of(wvs, cvs, v._0) } From 52ddfef2b2a46399d147241cb997c17b12d624ef Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 20:59:49 -0700 Subject: [PATCH 13/18] dasLLAMA: the bf16 arc's ledger entry, rows 240 and 242 The 2026-10-25 entry carries the three boards' re-measured Qwen3-ASR rows at 3f7134f8f beside the previous ones, the Xeon's stage split, every in-tree race with its command (harness/cpu_asr_race.das's bf16, mel and kpass arms on the four boxes), the E4B pp512 trio and the decode under f16 and q8_0 K/V. Row 240 states what the family reaches (the conv weights are F16, the family has its AMX leg only) and what it owes; row 242 asks for the ASR session's CPU K/V codec ruling. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/PERF_LEDGER.md | 43 ++++++++++++++++++++++++++++ modules/dasLLAMA/followup_general.md | 41 +++++++++++++++----------- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/modules/dasLLAMA/PERF_LEDGER.md b/modules/dasLLAMA/PERF_LEDGER.md index 99892c34e3..5c8a51dc05 100644 --- a/modules/dasLLAMA/PERF_LEDGER.md +++ b/modules/dasLLAMA/PERF_LEDGER.md @@ -10,6 +10,49 @@ and coverage; this ledger is the backlog for the perf pass that follows them. Ev what it costs today and what the fix would change. ## Entries +- **MEASURED (2026-10-25, `debug-jit`, `direction-grade`) - the float GELU, the bf16 plane walk, the AMX bf16 weight + tile, the mel's lanes and the decoder's K/V bytes, on the three AWS boards.** Every figure here is a `-jit` script + run (the three boards' rows excepted), `DAS_TUNE_POLICY` unset, `DAS_TUNE_MANIFEST` the box's rig release sidecar + under `performance/_rig/dasllama-bench/` (the M5: `performance/m5.tune.json`), `DASLLAMA_ALLOW_UNTUNED=1`, branch + `bbatkin/bf16-tower-gemm` at the commit named. The boards' Qwen3-ASR CPU rows at 3f7134f8f: each box's rig exe + re-released at that tip (`daspkg release --quick`, no policy override, the sidecar re-minted - the Granite Rapids + box's names `bf16w_tile_gen` = `dot_amx_bf16_width512_mr16_kstep1_nrsplit2`) and `performance/gen_bench_records.das + -- --workload asr -o Qwen3-ASR` run on it, the references `ARCHITECTURE_MEASUREMENT_AUDIO.md#asr-cpu-references`, + every cell the best of `--asr-reps` 2 (the exe prints a clip's best alone, `followup_general.md` row 241), das + against `llama-mtmd-cli` on gb1 (198.7 s), total / encode ms, every `was` the 2026-10-24 tile entry's ea6d942c2 row: + gnr (AWS m8i, Xeon 6975P-C, 8 cores) 13807 / 2007 against 20857 / 7481 (1.51x; was 15797 / 2917 against 21610 / + 7304), epyc9r14 (AWS m7a, 16 cores) 11694 / 2499 against 15916 / 8009 (1.36x; was 12784 / 3071 against 15981 / + 8051), graviton4 (AWS m8g, 16 cores) 8406 / 2462 against 67900 / 64309 (8.1x; was 9209 / 3168 against 67958 / + 64362); none published, the release re-mint owes the board (row 238). The Xeon's stage split + (`harness/asr_stage_probe.das -- -m Qwen3-ASR-0.6B-Q8_0.gguf --mmproj mmproj-Qwen3-ASR-0.6B-bf16.gguf -w gb1.wav -r + 1 --lang auto`, `JOBQUE_PROFILING=1 DAS_LOG_LEVEL=info -no-module-cache` for the engine buckets, gb1): at 7d7e39fc7 + (the GELU kernel and the bf16 walk) against ea6d942c2 (the 2026-10-24 tile entry's split) the encode 2451 against + 2924 ms, `enc.ffn.gelu` 41 against 194, `q3a.conv.mm` 1123 against 1446, `q3a.blocks` 588 against 726, `q3a.mel` 396 + against 403 (the mel's frame split is 258ca3799e; the race below carries it). `q3a.conv.mm` is GEMM-bound: 200 + chunks of a 3.3 GFLOP c2 and a 0.86 GFLOP c3 are 840 GFLOP at 750 GFLOP/s of f32 tile on 8 cores, against the race's + 1000 hot; the mmproj's conv weights are F16 and its blocks run the q8 lane, so the bf16 family reaches the conv + front only through an f16 ruling (row 240). `harness/cpu_asr_race.das -- --reps 5` (`-no-module-cache`, one process + a box, interleaved best of 5) at 24db4942af: the `bf16` arm, the bf16 plane through `matmul_bf16_batch` against the + f32 plane through `matmul_batch`, GFLOP/s at n d ntok 2560 10752 512 / 4320 480 800 / 1152 1152 256 - gnr on the + stamped AMX family 3921 / 3041 / 4093 against 782 / 923 / 1019; epyc9r14 on the widen walk 1667 / 1638 / 1537 + against 1610 / 1618 / 1618; graviton4 on the widen walk 664 / 599 / 657 against 599 / 563 / 636; the M5 Max on the + widen walk 978 / 846 / 672 against 988 / 786 / 761 (the dot_bf16 nest the walk replaces read 86 GFLOP/s on gnr at + the conv shape, the 2026-10-24 entry's nest arm). The `mel` arm (`log_mel_qwen3a` over a 60 s synthetic clip, 128 + mels; one lane / the lanes, ms): gnr 98.4 / 16.5, epyc9r14 101.0 / 15.1, graviton4 105.2 / 13.1, M5 52.2 / 10.2, 0 + of 768128 elements apart on every box. The `kpass` arm (the decode attention's K pass over 28 layers x 3000 keys x 8 + kv heads x 128, f16, two q heads a kv head, `dot_f16` a key a head; GB/s one lane / all lanes): (kv head, slice) + units gnr 3.6 / 30.3, epyc9r14 2.9 / 35.2, graviton4 8.4 / 87.5, M5 15.6 / 139.4; one slice over every kv head gnr + 9.1 / 62.1, epyc9r14 13.4 / 69.0, graviton4 9.6 / 102.0, M5 23.1 / 200.0. gemma-4-E4B Q8_0 pp512 on gnr + (`benchmarks/lcpp_bench.das -- -m gemma-4-E4B-it-Q8_0.gguf -p 512 -n 0 -r 3 -t 8 --for-debug-purposes`, the file + `unsloth/gemma-4-E4B-it-GGUF` at `resolve/main`): master 2d7d99fbf 284.7 +/- 18.3 tok/s (cv 6.4%, void), + 24e19e9916's kernels on the widen walk 359.8 +/- 0.3, the same with the AMX entry in the sidecar 368.8 +/- 0.2, the + frozen fixture's 8 opening tokens matching on all three. The decode on gnr at 24e19e9916's kernels (the stage probe, + the engine buckets on): f16 K/V (the ASR session's default) 11115 ms over 451 steps, `attn_chain` 8843 of it, + `blk_ffn` 1843, `mm_gemm` 525; `--kv q8` (the q8_0 K/V codec) 6285 ms over 443 steps, `attn_chain` 4097 - the + transcript moves (row 242). The chain's units re-ordered to one slice over every kv head (fcfb1dacd8) read + `attn_chain` 8942 ms against 24e19e9916's 8724 on gnr - within the run's noise; the kpass arm's two-fold does not + reach the chain, whose wall sits elsewhere. + - **MEASURED (2026-10-24, `direction-grade`) - the fp32 batch GEMM tile and the lane-split conv front on the three AWS boards.** `matmul_batch_core` runs a four-by-four tile (`f32_batch_tile`, `ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice`); `harness/f32_batch_race.das -- --reps 5` races its two diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 1762b24bab..91963bc76c 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -2295,22 +2295,19 @@ `bin/daslang modules/dasLLAMA/performance/gen_bench_records.das -- -w llm --backend cpu -o Qwen3.6-35B` there, `--ref-clean` naming the clean-cpu `llama-bench` `benchmarks/setup_lcpp_ref.das` builds at `DEFAULT_REF_SHA`. -240. **The bf16 towers multiply in f32, and the Qwen3-ASR decoder's attention over the audio context is unshared.** The - reference's encoders run bf16 x bf16 arithmetic (its mmproj weights stay bf16, its activations round to bf16, the - dot is the half-width one - `vdpbf16ps`, the AMX bf16 tile, NEON `bfdot`); ours widen every bf16 or f16 tower tensor - to f32 at load and run f32 x f32 - the Qwen3-ASR conv front (`conv2d_s2`, `dasllama_qwen3a.das`), the gemma4a and - Omni audio encoders, the vision towers - so on every mmproj line the encode pays twice the weight bytes and no - half-width dot (the 2026-10-24 `PERF_LEDGER.md` tile entry: the Xeon's encode is 2917 ms against the reference's - 7304 after the fp32 tile, with that arithmetic gap still in it). The Qwen3-ASR decoder spends its 451 steps (11.5 s - on gnr; the release-17 Zen4 row's total less its encode reads 9.6 s, the 2026-10-22 entry) in `attn_chain` and - `blk_attn` over the ~3000 audio tokens. Done = a `[tune]` family for the bf16-weight batch GEMM in the generator - (`dasllama_gemm_gen.das`: an f32-to-bf16 activation panel, `vdpbf16ps` on avx512bf16, `bfdot` on the arm bf16 - extension, the AMX bf16 tile on the x86-amx class, the fp32 tile the reference body and fallback, a bf16-class gate - in `harness/gen_tune_probe.das`), the bf16 tower planes served through it, and a decode attention step that shares - the audio-context K/V reads across heads, with the Qwen3-ASR conv front's first layer (nine taps a dot, n 9) and - its erf GELU (an exp in double a lane) served through the same family and a float-lattice GELU - the Qwen3-ASR and - image-chat CPU rows re-minted on every board box, - `bin/daslang modules/dasLLAMA/performance/gen_bench_records.das -- -w asr -o Qwen3-ASR` and `-w image` on each. +240. **The bf16 towers' conv fronts are F16, and the half-width family has only its AMX leg.** `bf16w_tile_gen` + (`dasllama_math_gen.das`) multiplies a native bf16 plane as the reference does - the activations rounded to bf16, + the TMUL bf16 tile over a k-pair panel - and `matmul_bf16_batch` reaches it where a sidecar names an `amx_bf16` + perm; the gemma-E per-layer projection and the bf16 vision patch planes ride it (the 2026-10-25 `PERF_LEDGER.md` + Granite Rapids entry: E4B pp512 359.8 -> 368.8 tok/s over the widen walk, the tile 3921 against the f32 plane's + 782 GFLOP/s at that shape). What is left: the Qwen3-ASR mmproj's conv weights are F16 (`a.conv2d.*.weight`, + `a.conv_out.weight`), its blocks run the q8 lane, so the conv front's 1123 ms of f32 tile on the Xeon reaches the + matrix unit only through an f16 operand - AMX-FP16 (Granite Rapids on, Sapphire Rapids off) or an f16-to-bf16 + panel (three mantissa bits under the reference's f16 x f16); and the family has no vector leg, so Zen4 + (`vdpbf16ps`, avx512bf16) and Graviton (`bfdot` / `bfmmla`, the arm bf16 extension) run the widen walk. Done = the + conv front's operand ruled and served through the family, `vdpbf16ps` and `bfdot` perms in `dasllama_gemm_gen.das` + over the same panel with the probe gating them, and the Qwen3-ASR and image-chat CPU rows re-minted on every board + box at the release re-mint. 241. **The bench exe's ASR and TTS cells print a clip's best of their reps alone.** `lcpp_bench --asr` and `--tts` keep the minimum over `-r` reps (`r.enc_ms = min(...)`, the sentence sums) and the record row carries that one wall, @@ -2318,3 +2315,15 @@ per-rep spread to give where `REVIEW_MEASUREMENT.md`'s served-turn rule asks for one. Done = the ASR and TTS cells print every rep's wall beside the best (`rep i: ` lines) and the record row's `tests` entry carries `ms_reps`, the recorder and `test_bench_records_schema.das` reading it back, so a ledger entry can quote the min and max. + +242. **The ASR decoder's attention wall on CPU is its K/V bytes, and the session's codec is f16.** On the Granite + Rapids board the Qwen3-ASR decode over the ~3000 audio tokens spends `attn_chain` 8.8 s of 11.1 s (451 steps) + under the f16 K/V codec the ASR session defaults to (`set_asr_kv`), and 4.1 s of 6.3 s under q8_0 (443 steps - the + transcript moves); in `harness/cpu_asr_race.das`'s K pass the box streams 30 GB/s over (kv head, slice) units and + 62 GB/s by slices over every kv head, and the chain re-ordered to the latter did not move (the 2026-10-25 + `PERF_LEDGER.md` Granite Rapids entry). The reference decodes its f16 K/V at about 32 ms a step (its 21610 ms gb1 + total less its 7304 ms encode over the 451 steps, the 2026-10-24 tile entry's `llama-mtmd-cli` figures) against + our 24.6 (11115 ms over 451). Done = a ruling on the ASR session's CPU codec - q8_0 by default with the parity + cells' bars re-cut on its transcripts, or f16 kept and the row closed - written into + `ARCHITECTURE_MEASUREMENT_AUDIO.md` and the facade's default, and the Qwen3-ASR CPU rows re-minted on every board + box at the release re-mint. From e2f5d06bd5f6ad63076256ec97d98e31d47c2ba3 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 21:05:19 -0700 Subject: [PATCH 14/18] dasLLAMA tests: the q8-against-f32 transcribe cells compare word lists The gemma4a cell held the two rails to the same text; the f32 rail now runs the bf16 plane through the tile walk, whose rounding splits a comma on a near-tie token, so both ASR cells read their transcripts through transcript_words, as the Metal cell already does. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/tests/CLAUDE.md | 3 ++- modules/dasLLAMA/tests/test_model_image.das | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index d341b5fd42..dd95490392 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -487,7 +487,8 @@ is its untied twin on TinyLlama-1.1B - the blob plane carrying the classifier AN embedding's linear q8 copy (`emb_q8` without `cls_q8`): the layout gate admits the copy, cold mint and warm map keep the q8 table with no fp32 copy, GPU serve, the same teacher-forced bar (`--arm metal` and `--arm untied` both select it - substring match). The ASR-family -arms (`parakeet` transcript-exact, `qwen3a`/`canary`/`gemma4a` element-exact planes, +arms (`parakeet` transcript-exact, `qwen3a`/`canary`/`gemma4a` element-exact planes - the gemma4a and qwen3a q8-against-f32 +transcribe cells comparing word lists, a comma riding a near-tie token between the two rails -, `canary-dec` = the opt-in fp32 Model rail, token-exact) re-save their images from cold each run like the voxtral arm. The canary arm carries both lanes: the f32 element-exact cell and the q8 cell (read-time transcode - qblob/qscales/compact-blob element-exact vs a staged read). diff --git a/modules/dasLLAMA/tests/test_model_image.das b/modules/dasLLAMA/tests/test_model_image.das index b8479ffbf5..62435471fc 100644 --- a/modules/dasLLAMA/tests/test_model_image.das +++ b/modules/dasLLAMA/tests/test_model_image.das @@ -2436,7 +2436,7 @@ def test_gemma4a_encoder_image_roundtrip(t : T?) { [test] def test_gemma4a_transcribe_q8_ab(t : T?) { - t |> run("gemma4a transcribe: the q8 tower vs the f32 rail on jfk, same text") @(t : T?) { + t |> run("gemma4a transcribe: the q8 tower vs the f32 rail on jfk, the same words") @(t : T?) { return if (!family_on(t, "gemma4a") || !arm_on(t, "gemma4a") || !jit_only(t)) let dec_path = path_join(models_dir(), "gemma-4-E2B-it-Q8_0.gguf") let mmp_path = path_join(models_dir(), "mmproj-gemma-4-E2B-it-bf16.gguf") @@ -2471,7 +2471,8 @@ def test_gemma4a_transcribe_q8_ab(t : T?) { } to_log(LOG_INFO, "gemma4a q8 : {q8_text}\n") to_log(LOG_INFO, "gemma4a f32: {f32_text}\n") - t |> equal(q8_text, f32_text, "q8 tower transcript matches the f32 rail") + // the words, not the punctuation: a comma rides a near-tie token the two rails' rounding can split + t |> equal(join(transcript_words(q8_text), " "), join(transcript_words(f32_text), " "), "q8 tower transcript matches the f32 rail, punctuation aside") delete samples } } @@ -2699,7 +2700,7 @@ def test_qwen3a_tower_q8_ab(t : T?) { [test] def test_qwen3a_transcribe_q8_ab(t : T?) { - t |> run("qwen3a transcribe: the q8 tower core vs the f32 rail on jfk, same text") @(t : T?) { + t |> run("qwen3a transcribe: the q8 tower core vs the f32 rail on jfk, the same words") @(t : T?) { return if (!family_on(t, "qwen3a") || !arm_on(t, "qwen3a") || !jit_only(t)) let dec_path = path_join(models_dir(), "Qwen3-ASR-0.6B-Q8_0.gguf") let mmp_path = path_join(models_dir(), "mmproj-Qwen3-ASR-0.6B-bf16.gguf") @@ -2735,7 +2736,8 @@ def test_qwen3a_transcribe_q8_ab(t : T?) { } to_log(LOG_INFO, "qwen3a q8 : {q8_text}\n") to_log(LOG_INFO, "qwen3a f32: {f32_text}\n") - t |> equal(q8_text, f32_text, "q8 tower transcript matches the f32 rail") + // the words, not the punctuation: a comma rides a near-tie token the two rails' rounding can split + t |> equal(join(transcript_words(q8_text), " "), join(transcript_words(f32_text), " "), "q8 tower transcript matches the f32 rail, punctuation aside") delete samples } } From 874316340fa6c92ce1a3756b87bc2dcb9b4c4f87 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 21:10:34 -0700 Subject: [PATCH 15/18] dasLLAMA: the measurement rules define board box, re-mint and held entry The cold re-read's batch: the three terms move to the definitions block, the fallback rule names the defined "can mint", the new-code rule writes its figures into the ledger like its siblings, the per-box re-mint reads as it runs, and the withdrawn-row rule folds into the changed-cell rule it repeated. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/REVIEW_MEASUREMENT.md | 59 ++++++++++++++------------ 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 935aeb4ec1..8f9b04ee19 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -66,12 +66,20 @@ A cell, script or exe of this repository is a spawned or manual cell, a script i diff's tip, or an exe built from that tree; a script outside the tree is not one, whatever it calls. -A box can mint a spawned cell when a rig leg it neither refuses nor skips drives the path the owed -row measures and `performance/gen_bench_records.das --catalog official` carries the model; a -manual cell when it runs the documented command's OS and backend and holds its model; a cell owed -for new code when a rig leg or a documented command reaches that code's input. A row owed for a -route that no rig leg drives and no `PROFILE.md` section documents a command for is one no box can -mint. +A box can mint a figure from a spawned cell when a rig leg it neither refuses nor skips drives the +path the owed figure measures and `performance/gen_bench_records.das --catalog official` carries the +model; from a manual cell when it runs the documented command's OS and backend and holds its model; +a figure owed for new code when a rig leg or a documented command reaches that code's input. A +figure owed for a route that no rig leg drives and no `PROFILE.md` section documents a command for +is one no box can mint. + +A board box is a box with a `performance/records/.json`. The release re-mint is a diff that +runs `performance/gen_bench_records.das` with a release's exe on one or more board boxes and writes +their rows into `performance/records/.json`. + +A `held` entry is a `performance/records/annotations.json` entry carrying a board row's `gguf`, +`box`, `engine`, `backend`, `flavor` and `workload`, whose `held` field names the follow-up ledger +row that keeps the row off the board. **The naming that a rule governing a figure asks for attaches to a figure only by a sentence, a table heading, a provenance line or a `PERF_LEDGER.md` entry's opening paragraph that @@ -166,14 +174,16 @@ entry - that figure goes in the PR that lands the code.** **A diff that lets a served turn, or a media encode a turn takes (an image, a video frame or an audio chunk), reach code no spawned or manual cell's input reaches - a new dispatch arm, a window-size arm inside a path a cell drives at other sizes, or a changed default route - adds to -`performance/gen_bench_records.das` a cell whose input reaches it, in the same change, and names in -its PR body the `followup_general.md` row under which the release re-mint owes its rows.** +`performance/gen_bench_records.das` a cell whose input reaches it, writes the new cell's figures +from every board box of the author's that can mint them into a dated `PERF_LEDGER.md` entry, in the +same change, and names in its PR body the `followup_general.md` row under which the release re-mint +owes its rows.** -**A change that owes a cell no box of the author's can run, or a ledger figure no box of the -author's can measure, names instead, in the same change, an artifact that shows the new code ran end -to end - a record, a gate output, an instrument's print - or the profile output that shows the -change is faster (`benchmarks/lcpp_bench.das`'s `forward_profile` rows, or the tier's -`DASLLAMA_GPU_PROF=1` token ledger), with the flags that run took.** +**A change that owes a ledger figure no box of the author's can mint names instead, in the same +change, an artifact that shows the new code ran end to end - a record, a gate output, an +instrument's print - or the profile output that shows the change is faster +(`benchmarks/lcpp_bench.das`'s `forward_profile` rows, or the tier's `DASLLAMA_GPU_PROF=1` token +ledger), with the flags that run took.** **A diff that changes `dasllama/dasllama_metal_kernels.das` (the kernel library `metal_decode_init` compiles), `performance/profile_common.das`, or a `dasllama/` module @@ -184,20 +194,18 @@ run printed none.** A fat exe is what `daspkg release --fat ` builds (`DA **A diff whose PR body, or a line it adds or changes, states that a route the tree served before the diff runs faster or does less work, where a rig leg runs the changed lines, writes that leg's walls -from every board box of the author's that runs it - a board box is one with a -`performance/records/.json` - into a dated `PERF_LEDGER.md` entry naming the exe's commit and -the command, and names in the PR body the `followup_general.md` row under which the release re-mint -- `performance/gen_bench_records.das` run over every board box with a release's exe - owes every -board box's row; no diff but the release re-mint writes those rows into +from every board box of the author's that runs it into a dated `PERF_LEDGER.md` entry naming the +exe's commit and the command, and names in the PR body the `followup_general.md` row under which the +release re-mint owes every board box's row; no diff but the release re-mint writes those rows into `performance/records/.json`.** The board is the module's committed record of what a release's exe costs to serve; a kernel win that lands in no ledger entry is invisible to the next regression check, and a row minted between releases shows a speed no shipped binary gives (`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`). -**The release re-mint re-mints every row a `followup_general.md` row owes it in -`performance/records/.json`, deletes any `held` entry holding one, names each row in its PR -body, and puts each changed cell's last board figure beside the new one in a dated `PERF_LEDGER.md` -entry in the same change.** +**The release re-mint re-mints, on each board box it runs on, every row a `followup_general.md` row +owes it there, deletes any `held` entry holding one, names each row in its PR body, and puts each +changed cell's last board figure beside the new one in a dated `PERF_LEDGER.md` entry in the same +change.** **A diff whose PR body, or a line it adds or changes, states that a route the tree served before the diff runs faster or does less work, where no rig leg runs the changed lines, states in its PR body @@ -213,12 +221,9 @@ the old ones, and names in its PR body the `followup_general.md` row under which owes the cell's rows; no diff but the release re-mint writes those rows into `performance/records/.json`.** -**A withdrawn row of a changed board cell is deleted from `performance/records/.json` or gets -a `held` entry - a `performance/records/annotations.json` entry carrying the row's `gguf`, `box`, -`engine`, `backend`, `flavor` and `workload`, whose `held` field names the follow-up ledger row that -keeps it off the board - and stays recorded.** -**A `PERF_LEDGER.md` entry that withdraws an earlier entry's figure states the withdrawal's reason.** +**A `PERF_LEDGER.md` entry that withdraws an earlier entry's figure states the withdrawal's +reason.** **A rate or wall of any served-turn leg, or a ratio of two such rates or walls, written down as a measurement rather than as a prediction is a defect unless it cites the committed board row it From 00369153334b6e0cb22b6fe5664c603b498d576d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 21:48:32 -0700 Subject: [PATCH 16/18] dasLLAMA: the bf16w family on the shared walk, one activation builder The review round's folds: the family's two companions were byte copies of the kq tileform and q8q8 tokstep keys, so they name those; bf16w_batch and the probe walk one bf16w_walk over the engine's bf16_walk; xbf16_build is generic over its source and builds the f32 plane too, the widened copy a widen_bf16 pass; bf16_bits is the module's rounding, the harness copies gone; bf16w_batch_serves is the gate the test asks instead of restating it. The tuner's two tile benches and its two in-place unary benches fold into one each, the GELU oracle's erf series lives once in _compares. cpu_asr_race rides _race_fixture, gains the GELU arm (the double body raced as its own arm) and the bf16 plane's row nest and widen walk as candidates. The transcribe cells' word compare has a control and logs both lists; the one-lane mel cell holds the threaded mel to its image; the probe's bf16w bar is the envelope on every row, the reference row's tails running the box's f32 tile perm. The three checklists carry their self-review batches; the two tests' comments on the GELU say what the kernel is now. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/REVIEW.das | 2 +- modules/dasLLAMA/REVIEW.md | 46 ++--- modules/dasLLAMA/REVIEW_MEASUREMENT.md | 45 ++--- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 8 - modules/dasLLAMA/dasllama/dasllama_math.das | 1 + .../dasLLAMA/dasllama/dasllama_math_gen.das | 90 +++++----- modules/dasLLAMA/harness/cpu_asr_race.das | 165 +++++++++++------- modules/dasLLAMA/harness/gen_tune_probe.das | 48 ++--- modules/dasLLAMA/harness/tune_kernels.das | 165 ++++++------------ modules/dasLLAMA/tests/CLAUDE.md | 5 +- modules/dasLLAMA/tests/REVIEW.md | 28 +-- modules/dasLLAMA/tests/_compares.das | 13 ++ modules/dasLLAMA/tests/test_matmul_batch.das | 7 +- .../tests/test_metal_misc_kernels.das | 2 +- modules/dasLLAMA/tests/test_model_image.das | 25 ++- .../dasLLAMA/tests/test_tower_asr_kernels.das | 12 +- .../tests/test_vulkan_tower_kernels.das | 6 +- modules/dasLLAMA/tests/test_whisper.das | 26 +++ 18 files changed, 345 insertions(+), 349 deletions(-) diff --git a/modules/dasLLAMA/REVIEW.das b/modules/dasLLAMA/REVIEW.das index bcdbf144b2..5563198c5f 100644 --- a/modules/dasLLAMA/REVIEW.das +++ b/modules/dasLLAMA/REVIEW.das @@ -1799,7 +1799,7 @@ let private GEMM_GEN_FILES = [ "modules/dasLLAMA/dasllama/dasllama_gemm_gen.das", "modules/dasLLAMA/dasllama/dasllama_gemm_schema.das" ] -let private GEMM_GEN_STAMP_HASH = 0x151a4437a5d9c67ul +let private GEMM_GEN_STAMP_HASH = 0x13a6a4a55671fefaul [arch(at = "ARCHITECTURE_ENGINE_FORMATS.md#cpu-kernel-tiers")] def private check_gemm_gen_stamp { diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 32db9fcf89..d6eb63bcb6 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -15,9 +15,9 @@ rewrites the problem a `followup_*.md` row states, adds a file under `dasllama/` STYLE037/STYLE038 suppression anywhere under this folder applies `REVIEW_DOCS.md` (beside this file) too.** -**A diff that adds, changes or drops a check in a `REVIEW.das` under this folder, or adds a name -to a check's licensed set - the names a check does not flag - applies `REVIEW_GATES.md` (beside -this file) too.** +**A diff that adds, changes or drops a check in a `REVIEW.das` under this folder, adds a name to a +check's licensed set - the names a check does not flag - or changes a constant a check compares +against, applies `REVIEW_GATES.md` (beside this file) too.** **Wherever it lives, each of these answers to this folder's `benchmarks/REVIEW.md` beside its own folder's checklist: code that times a run on its own clock (not a spawned child's) and hands the @@ -182,16 +182,18 @@ Every recorded row, tune sidecar and exchange entry carries the release, so a bu on another module's hook, moves a family encode stage (a `dasllama/dasllama_.das` stage turning input into embeddings or filling a cache the next stage reads) onto a GPU hook, calls a `set_*_q8` lane setter under `dasllama/` or `harness/`, adds or changes an override (an environment -knob, a setter only code outside `dasllama/` sets, or a state file a run reads, that moves a gate, -policy or threshold off its default), or adds or changes an announce (the line a run prints where -an override changes the outcome) applies `REVIEW_OVERRIDES.md` (beside this file) too.** - -**A diff that cites a timing (in a `PERF_LEDGER.md` row, a `followup_*.md` row, a commit message -or the PR body) as the reason it picks one candidate over another - a constant's value, a formula, -or a predicate choosing among kernels that compute the same result - takes the winner from one -script that races every candidate in one process, interleaved, each chosen by a flag or argument -in the code the diff ships** - two commits or two processes differ by everything else that -changed, so their timings pick nothing. +knob, a setter only code outside `dasllama/` sets, or a state file a run reads (a tune sidecar is +not one: it names kernel perms, which the tune framework's own gates rule), that moves a gate, +policy or threshold off its default), or adds or changes an announce (the line a run prints where an +override changes the outcome) applies `REVIEW_OVERRIDES.md` (beside this file) too.** + +**A diff that cites a timing (in a `PERF_LEDGER.md` row, a `followup_*.md` row, a commit message or +the PR body) as the reason it picks one candidate over another - a constant's value, a body +replacing another that computes the same quantity, a predicate or a dispatch order choosing among +kernels that compute the same result - takes the winner from one script that races every candidate +in one process, interleaved, each chosen by a flag or argument in the code the diff ships, or +carried as an arm of the race script where the diff deletes it** - two commits or two processes +differ by everything else that changed, so their timings pick nothing. **A diff that picks a candidate by racing every candidate in one process, interleaved, puts the race's output lines, one per candidate and each naming it, in the PR body or in a `PERF_LEDGER.md` @@ -203,21 +205,21 @@ at least one value on each side of it that the device admits, in that race.** **A new call site of a CPU or GPU matrix multiply whose weight operand is an f32 plane, outside a correctness-comparison path (one whose only job is to produce a reference result to check another -against), where the same backend already serves the same weights and shape through a kernel on a -quantized or f16 form of them, is a defect - call that kernel instead.** A site that must stay f32 -for another reason is ledgered on its file's charter line (the line, in the companion that -`ARCHITECTURE.md#file-charters` routes to, that says what the file holds); a comment at the call -site does not discharge this. +against) or a timing race's reference arm, where the same backend already serves the same weights +and shape through a kernel on a form of them other than f32, is a defect - call that kernel +instead.** A site that must stay f32 for another reason is ledgered on its file's charter line (the +line, in the companion that `ARCHITECTURE.md#file-charters` routes to, that says what the file +holds); a comment at the call site does not discharge this. **A caller and its callee never both check one guard: the caller keeps the check and the callee drops it when the callee is `[cold_path]`; where the caller's check also gates its own work, the caller computes the decision once and passes it to the callee; otherwise the caller's copy goes.** An edit to either copy leaves the caller testing a condition the callee no longer applies. -**Logic or a named constant that two files in one folder both use, and that neither the language -nor the test contract forces them to restate (one predicate written over an enum and again over -its int value; a test's CPU reference copy of the arithmetic it checks), lands once - in a file -both already require, or in a new file both require when they share none - never as a second +**Logic or a named constant that two files in one folder both use, and that neither the language nor +the test contract forces them to restate (a test's CPU reference copy of the arithmetic it checks is +forced; one predicate written over an enum and again over its int value is not), lands once - in a +file both already require, or in a new file both require when they share none - never as a second copy.** Two spellings drift apart on the first edit to one. **A piece that two folders both need, neither containing the other, lands in the folder that owns diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 8f9b04ee19..597b2cf5cc 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -81,6 +81,9 @@ A `held` entry is a `performance/records/annotations.json` entry carrying a boar `box`, `engine`, `backend`, `flavor` and `workload`, whose `held` field names the follow-up ledger row that keeps the row off the board. +**A diff that owes a figure from every board box of the author's lists the author's board boxes in +its PR body.** + **The naming that a rule governing a figure asks for attaches to a figure only by a sentence, a table heading, a provenance line or a `PERF_LEDGER.md` entry's opening paragraph that unambiguously covers it, or by a citation of a passage or board row whose provenance covers @@ -173,11 +176,10 @@ entry - that figure goes in the PR that lands the code.** **A diff that lets a served turn, or a media encode a turn takes (an image, a video frame or an audio chunk), reach code no spawned or manual cell's input reaches - a new dispatch arm, a -window-size arm inside a path a cell drives at other sizes, or a changed default route - adds to -`performance/gen_bench_records.das` a cell whose input reaches it, writes the new cell's figures -from every board box of the author's that can mint them into a dated `PERF_LEDGER.md` entry, in the -same change, and names in its PR body the `followup_general.md` row under which the release re-mint -owes its rows.** +window-size arm inside a path a cell drives at other sizes, or a changed default route - adds a +spawned cell whose input reaches it, writes the new cell's figures from every board box of the +author's that can mint them into a dated `PERF_LEDGER.md` entry, in the same change, and names in +its PR body the `followup_general.md` row under which the release re-mint owes its rows.** **A change that owes a ledger figure no box of the author's can mint names instead, in the same change, an artifact that shows the new code ran end to end - a record, a gate output, an @@ -194,19 +196,21 @@ run printed none.** A fat exe is what `daspkg release --fat ` builds (`DA **A diff whose PR body, or a line it adds or changes, states that a route the tree served before the diff runs faster or does less work, where a rig leg runs the changed lines, writes that leg's walls -from every board box of the author's that runs it into a dated `PERF_LEDGER.md` entry naming the +from every board box of the author's that can mint it into a dated `PERF_LEDGER.md` entry naming the exe's commit and the command, and names in the PR body the `followup_general.md` row under which the -release re-mint owes every board box's row; no diff but the release re-mint writes those rows into -`performance/records/.json`.** The board is the module's committed record of what a release's -exe costs to serve; a kernel win that lands in no ledger entry is invisible to the next regression -check, and a row minted between releases shows a speed no shipped binary gives +release re-mint owes every board box's row.** The board is the module's committed record of what a +release's exe costs to serve; a kernel win that lands in no ledger entry is invisible to the next +regression check, and a row minted between releases shows a speed no shipped binary gives (`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`). **The release re-mint re-mints, on each board box it runs on, every row a `followup_general.md` row owes it there, deletes any `held` entry holding one, names each row in its PR body, and puts each -changed cell's last board figure beside the new one in a dated `PERF_LEDGER.md` entry in the same +re-minted row's last board figure beside the new one in a dated `PERF_LEDGER.md` entry in the same change.** +**No diff but the release re-mint writes a row into `performance/records/.json`; a figure any +other diff owes lands in a dated `PERF_LEDGER.md` entry.** + **A diff whose PR body, or a line it adds or changes, states that a route the tree served before the diff runs faster or does less work, where no rig leg runs the changed lines, states in its PR body that no rig leg runs them and names the instrument that does, with its command line.** @@ -216,20 +220,19 @@ the model components (encoder, decoder, tower) it runs or the backend that serve or rows it divides the wall by, or the reference build it times (`DEFAULT_REF_SHA` in `benchmarks/setup_lcpp_ref.das`, or anything else deciding which reference binary or environment the run measures) - deletes each affected row or gives it a `held` entry, writes the cell's new figures -from every board box of the author's that runs the cell into a dated `PERF_LEDGER.md` entry beside -the old ones, and names in its PR body the `followup_general.md` row under which the release re-mint -owes the cell's rows; no diff but the release re-mint writes those rows into -`performance/records/.json`.** - +from every board box of the author's that can mint it into a dated `PERF_LEDGER.md` entry beside the +old ones, and names in its PR body the `followup_general.md` row under which the release re-mint +owes the cell's rows.** **A `PERF_LEDGER.md` entry that withdraws an earlier entry's figure states the withdrawal's reason.** -**A rate or wall of any served-turn leg, or a ratio of two such rates or walls, written down as -a measurement rather than as a prediction is a defect unless it cites the committed board row it -came from, or names harness, flags, environment overrides, box, the exe or script that ran it, -and - for a figure aggregated over more than one timed run or input - the number of runs or -inputs and the spread of the per-run figures: the standard deviation, or the min and max.** +**A rate or wall of any served-turn leg, or a ratio of two such rates or walls, written down as a +measurement rather than as a prediction is a defect unless it cites the committed board row it came +from, or names harness, flags, environment overrides, box, the exe or script that ran it, and - for +a figure aggregated over more than one timed run or input - the number of runs or inputs and the +spread of the per-run figures: the standard deviation, or the min and max - or, where the instrument +prints no spread, the `followup_general.md` row owing it.** **A diff that records a measured number an instrument prints - a time, a rate, or a figure computed from one - or changes what such a number measures, ships that number's alternate in the diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 73d0783a72..04359f37fe 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -3074,12 +3074,6 @@ def private bf16w_tile_gen(var gc : LlvmCodeCtx) : bool { return emit_amx_bf16_tile(gc, p) } -def private bf16w_tokstep_gen(var gc : LlvmCodeCtx) : bool => - int_companion(gc) $(p) => dot_kind(p.dotPrim) == DOT_AMX_BF16 ? 32 : 4 - -def private bf16w_tileform_gen(var gc : LlvmCodeCtx) : bool => - int_companion(gc) $(p) => dot_kind(p.dotPrim) == DOT_AMX_BF16 ? p.nrsplit : 0 - def private is_witness_signature(fn : Function const?) : bool => nullary_signature(fn, Type.tBool) // The witness: `() : bool` — did the stamped family actually EMIT on this target? Reference @@ -5027,8 +5021,6 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::kq_witness", @@witness_gen) register_llvm_code_generator("dasllama_gemm_gen::kq_tileform", @@tileform_gen) register_llvm_code_generator("dasllama_gemm_gen::bf16w_tile", @@bf16w_tile_gen) - register_llvm_code_generator("dasllama_gemm_gen::bf16w_tokstep", @@bf16w_tokstep_gen) - register_llvm_code_generator("dasllama_gemm_gen::bf16w_tileform", @@bf16w_tileform_gen) register_llvm_code_generator("dasllama_gemm_gen::q8_panel", @@q8_panel_gen) register_llvm_code_generator("dasllama_gemm_gen::q8_panel_s16", @@q8_panel_s16_gen) for (f in type) { diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 37fc2a5da3..6dd9bc708a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -909,6 +909,7 @@ var private g_bf16_batch_gen : Bf16BatchGenFn def set_bf16_batch_gen(fn : Bf16BatchGenFn) { g_bf16_batch_gen = fn } +def get_bf16_batch_gen() : Bf16BatchGenFn => g_bf16_batch_gen // the one body: the float-override seam then the core. Both weight forms below reach it with a // base pointer, so a borrowed plane and an owned blob take the identical route. diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index b578adb5a6..4e08e1e857 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -16,6 +16,7 @@ require daslib/tune public // [tune]/[tune_perm]/[tune_companion] + [llvm_code] require [tune_framework] public // PUBLIC so tune_status reaches app roots that require this module; the one framework call below sits under static_if require dasllama/dasllama_tune_scope // nolint:STYLE029,LINT019 — an ORDER dependency, not a symbol use: the dasllama [tune_scope] must be on the AST before this module's [tune] kernels pick their perms require dasllama/dasllama_gemm_schema +require dasllama/dasllama_convert // widen_bf16 - the bf16w family's rounded activations widened for its tails require daslib/jobque_boost public require dasllama/dasllama_par require dasllama/dasllama_env // DASLLAMA_ALLOW_UNTUNED wires tune_suppress_mint at [init] @@ -3572,7 +3573,7 @@ def private kq_tile4_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const? } //! bf16 round-to-nearest-even of an f32 bit pattern; the operands are finite products, a NaN payload is not preserved -def private bf16_bits(f : float) : uint16 { +def bf16_bits(f : float) : uint16 { let u = unsafe(reinterpret(f)) return uint16((u + 0x7FFFu + ((u >> 16u) & 1u)) >> 16u) } @@ -3627,7 +3628,12 @@ def kq_walk_panel_bytes(fmt : int; n, mr : int64) : int64 { return kq_reads_packed_planes(int64(fmt)) ? 0l : mr * (n / KQ_SB_ELEMS) * KQ_SB_ELEMS } -def xbf16_build(xqp : int8 const?; xsp : float const?; n, ntok, bsz : int64) : uint8 const? { +def private x_as_float(v : int8) : float => float(v) +def private x_as_float(v : float) : float => v + +//! the activation plane as bf16 bits in the 64-aligned scratch: token t's element i is `x[t * n + i]` +//! times its block scale `xsp[t * nblk + i / bsz]`, or the element itself under a null `xsp` +def xbf16_build(xqp : auto const?; xsp : float const?; n, ntok, bsz : int64) : uint8 const? { let nblk = n / bsz g_kq_xbf16 |> overwrite_resize(ntok * n + 32l) unsafe { @@ -3640,9 +3646,9 @@ def xbf16_build(xqp : int8 const?; xsp : float const?; n, ntok, bsz : int64) : u let srow = xsp + int64(t) * nblk var hrow = hx + int64(t) * n for (bi in range64(nblk)) { - let s = srow[bi] + let s = xsp == null ? 1.0 : srow[bi] for (i in range64(bi * bsz, bi * bsz + bsz)) { - hrow[i] = bf16_bits(float(xrow[i]) * s) + hrow[i] = bf16_bits(x_as_float(xrow[i]) * s) } } } @@ -4020,8 +4026,8 @@ def bf16w_witness_gen() : bool => false [tune_perm(mr = 16), tune_perm(dot = "amx_bf16", width = 512, mr = 16, kstep = 1, nrsplit = 2, requires = "amx-bf16,avx512bf16,avx512vnni"), tune_perm(dot = "amx_bf16", width = 512, mr = 16, kstep = 1, nrsplit = 1, requires = "amx-bf16,avx512bf16,avx512vnni"), - tune_companion(fn = "bf16w_tokstep_gen", gen = "dasllama_gemm_gen::bf16w_tokstep"), - tune_companion(fn = "bf16w_tileform_gen", gen = "dasllama_gemm_gen::bf16w_tileform"), + tune_companion(fn = "bf16w_tokstep_gen", gen = "dasllama_gemm_gen::q8q8_tokstep"), + tune_companion(fn = "bf16w_tileform_gen", gen = "dasllama_gemm_gen::kq_tileform"), tune_companion(fn = "bf16w_amx_cfg_gen", gen = "dasllama_gemm_gen::kq_amx_cfg"), tune_companion(fn = "bf16w_witness_gen", gen = "dasllama_gemm_gen::kq_witness"), tune(gen = "dasllama_gemm_gen::bf16w_tile", fallback = "mr16"), @@ -4047,36 +4053,19 @@ def bf16w_tile_gen(var yp : float?; panel : uint8 const?; _sp : float const?; xb } } -var private @scratch @exact_size g_bf16w_x : array var private @scratch @exact_size g_bf16w_xr : array -//! the bf16 activation plane: `ntok` rows of `n`, each f32 rounded to nearest even, 64-byte aligned; -//! `xr` receives the same rounded values widened back to f32, the tails' operand, so every output of -//! one call multiplies the same activation bits +//! an f32 activation plane rounded to bf16 for the tile, and in `xr` the same rounded values widened +//! back to f32 for the walk's tails, so every product of one call sees one rounding def xbf16_build_f32(xp : float const?; n, ntok : int64; var xr : float?&) : uint8 const? { - g_bf16w_x |> overwrite_resize(ntok * n + 32l) + let hx = xbf16_build(xp, null, n, ntok, n) g_bf16w_xr |> overwrite_resize(ntok * n) unsafe { - var hx = bf16_align64(addr(g_bf16w_x[0])) var rp = addr(g_bf16w_xr[0]) - let jobs = is_job_que_available() ? int(min(ntok, 16l)) : 1 - maybe_parallel_for(0, int(ntok), jobs) $(tb, te) { - unsafe { - for (t in range(tb, te)) { - let xrow = xp + int64(t) * n - var hrow = hx + int64(t) * n - var rrow = rp + int64(t) * n - for (i in range64(n)) { - let h = bf16_bits(xrow[i]) - hrow[i] = h - rrow[i] = reinterpret(uint(h) << 16u) - } - } - } - } + widen_bf16(rp, reinterpret(hx), ntok * n) xr = rp - return reinterpret(hx) } + return hx } //! `groups` sixteen-row groups from row g0 * 16 of a bf16 plane as the tile's k-pair rows: group gi's @@ -4096,18 +4085,24 @@ def bf16w_panel_build(wp : uint16 const?; n, g0, groups : int64; var dst : uint1 } } +//! whether `bf16w_batch` takes the shape on this box: a stamped tile form, a witnessed AMX bf16 unit, +//! n of whole 32s, d of a tile or more, ntok of a `tokstep` block or more +def bf16w_batch_serves(n, d, ntok : int64) : bool { + let form = int64(bf16w_tileform_gen()) + if (form <= 0l || n % 32l != 0l || d < 16l || !bf16w_witness_gen()) return false + return ntok >= int64(bf16w_tokstep_gen()) +} + //! the bf16-weight batch GEMM through the stamped tile family, units of `tileform` groups over whole //! `tokstep` blocks, every tail through the widen walk over the same bf16-rounded activations; false //! where the family declines or the shape is off its lattice (n % 32, d under 16, ntok under a tile) [hot_path, arch(at = "../ARCHITECTURE_CPU_KERNELS.md#bf16-weight-tile-family")] def bf16w_batch(var yp : float?; wp : uint16 const?; xp : float const?; n, d, ntok : int64) : bool { + if (!bf16w_batch_serves(n, d, ntok)) return false let form = int64(bf16w_tileform_gen()) - if (form <= 0l || n % 32l != 0l || d < 16l || !bf16w_witness_gen()) return false let ts = int64(bf16w_tokstep_gen()) - if (ntok < ts) return false let ng = d / 16l - let nu = int((ng + form - 1l) / form) - let tbs = (ntok / ts) * ts + let nu = bf16_units(ng, form) var xr : float? let xb = xbf16_build_f32(xp, n, ntok, xr) let xrc = unsafe(reinterpret(xr)) @@ -4120,22 +4115,8 @@ def bf16w_batch(var yp : float?; wp : uint16 const?; xp : float const?; n, d, nt bf16w_amx_cfg_gen() unsafe { var pp = reinterpret(pbase + int64(slot) * pstride) - for (u in range(ub, ue)) { - let g0 = int64(u) * form - let gn = min(form, ng - g0) - if (gn == form) { - bf16w_panel_build(wp, n, g0, form, pp) - var tk = 0l - while (tk < tbs) { - bf16w_tile_gen(myp, reinterpret(pp), null, reinterpret(xb), null, null, n, d, g0, tk) - tk += ts - } - if (tbs < ntok) { - batch_tile_block(myp, wp, xrc, n, d, g0 * 16l, (g0 + form) * 16l, tbs, ntok, ntok - tbs) - } - } else { - batch_tile_block(myp, wp, xrc, n, d, g0 * 16l, (g0 + gn) * 16l, 0l, ntok, ntok) - } + bf16w_walk(myp, wp, xrc, n, d, int64(bf16_unit_group(ub, form, ng)), int64(bf16_unit_group(ue, form, ng)), form, ntok, ts, pp) $(g, tk : int64) { + bf16w_tile_gen(myp, reinterpret(pp), null, reinterpret(xb), null, null, n, d, g, tk) } } } @@ -4145,6 +4126,19 @@ def bf16w_batch(var yp : float?; wp : uint16 const?; xp : float const?; n, d, nt return true } +//! the family's walk over row groups [g0, g1) and every token, one lane: the shared bf16 walk with the +//! family's panel, the tile the caller names, and the widen walk over the rounded activations `xrc` for +//! the token tail (four tokens a group) and the group tail short of a unit (a token at a time) +def bf16w_walk(var yp : float?; wp : uint16 const?; xrc : float const?; n, d, g0, g1, form, ntok, ts : int64; var pp : uint16?; tile : block<(g, tk : int64) : void>) { + bf16_walk(g0, g1, form, 0l, ntok, ntok, ts, $(g : int64) { + bf16w_panel_build(wp, n, g, form, pp) + }, tile, $(g, tk : int64) { + batch_tile_block(yp, wp, xrc, n, d, g * 16l, (g + 1l) * 16l, tk, tk + 4l, 4l) + }, $(g, gn, tk : int64) { + batch_tile_block(yp, wp, xrc, n, d, g * 16l, (g + gn) * 16l, tk, tk + 1l, 1l) + }) +} + //! the bf16 walk's token block: the knob or the floor, whichever is larger, in whole tiles and under no L2 clamp - the walk builds a row group's panel once per block, so a block the clamp cuts pays a second panel and a token tail def bf16_token_block() : int64 { let ts = amx_bf16_tile_tokens() diff --git a/modules/dasLLAMA/harness/cpu_asr_race.das b/modules/dasLLAMA/harness/cpu_asr_race.das index 4394511951..d52d67f342 100644 --- a/modules/dasLLAMA/harness/cpu_asr_race.das +++ b/modules/dasLLAMA/harness/cpu_asr_race.das @@ -1,12 +1,14 @@ options gen2 options _dasllama_internal = true -// The CPU ASR line's three in-process races, one line a candidate, best of --reps interleaved: -// --arm bf16 the bf16 weight plane against the f32 plane through matmul_batch at the gemma-E -// projection's pp512 shape, the Qwen3-ASR conv's and a vision patch class - the stamped -// bf16w family where the sidecar names a perm, the widen walk elsewhere -// --arm mel log_mel_qwen3a over a 60 s synthetic clip, one lane against the lanes, the two -// images compared element for element +// The CPU ASR line's four in-process races, one line a candidate, best of --reps interleaved rounds: +// --arm bf16 the bf16 weight plane's routes against the f32 plane through matmul_batch at the gemma-E +// projection's pp512 shape, the Qwen3-ASR conv's and a vision patch class: the stamped +// bf16w family where the sidecar names a perm, the widen walk, the row nest it replaced +// --arm gelu gelu_erf on the float lattice against the double erfc body it replaced, in place over a +// tower's FFN plane +// --arm mel log_mel_qwen3a over a 60 s synthetic clip, one lane against the lanes, the two images +// compared element for element // --arm kpass the decode attention's K pass over a model-sized f16 cache (28 layers x 3000 keys x // 8 kv heads x 128, two q heads a kv head): the (kv head, slice) unit order against one // slice over every kv head, one lane and all lanes @@ -20,6 +22,7 @@ require dasllama/dasllama_kv_codec require dasllama/dasllama_qwen3a require dasllama/dasllama_audio require dasllama/dasllama_par +require _race_fixture require daslib/jobque_boost require daslib/f16_cvt require daslib/clargs @@ -27,9 +30,9 @@ require math [CommandLineArgs] struct Args { - @clarg_doc = "Which race: bf16 | mel | kpass | all (default: all)" + @clarg_doc = "Which race: bf16 | gelu | mel | kpass | all (default: all)" arm : string = "all" - @clarg_doc = "Timed rounds a candidate, the best kept (default: 5)" + @clarg_doc = "Timed rounds a candidate, interleaved, the best kept (default: 5)" reps : int = 5 @clarg_short = "?" @clarg_name = "show-help" @@ -37,52 +40,84 @@ struct Args { help : bool } -def private bf16_bits_of(v : float) : uint16 { - let bits = unsafe(reinterpret(v)) - return uint16((bits + 0x7FFFu + ((bits >> 16u) & 1u)) >> 16u) +//! one candidate's line: its best round in ms, the median and the cv beside it, `rate` the figure the best makes +def private line(race, who : string; us : array; rate : block<(ms : float) : string>) { + let st = stats(us) + print("{race} {who}: {st.best} ms ({invoke(rate, st.best)}; median {st.median} ms, cv {st.cv})\n") } -def private best_ms(reps : int; blk : block<() : void>) : double { - var best = 1.0e30lf - for (_r in range(reps)) { - let t0 = ref_time_ticks() - invoke(blk) - best = min(best, double(get_time_usec(t0)) / 1000.0lf) - } - return best -} +//! past every dot length: the row nest serves every output +let private NEST_ONLY_MIN_N = 1l << 30 def private race_bf16(n, d, ntok : int64; reps : int) { - var w : array + let w <- race_fill(int(n * d), 1, 1.2, 0.0) var w16 : array - var x, y : array - w |> reserve_resize(int(n * d)) - w16 |> reserve_resize(int(n * d)) - x |> reserve_resize(int(ntok * n)) - y |> reserve_resize(int(ntok * d)) - for (i in range64(n * d)) { - w[i] = float(i % 13l) * 0.1 - 0.6 - w16[i] = bf16_bits_of(w[i]) - } - for (i in range64(ntok * n)) { - x[i] = float(i % 7l) * 0.2 - 0.7 - } - matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) // warm - matmul_batch(y, w, x, n, d, ntok) - var bf = 1.0e30lf - var ff = 1.0e30lf - for (_r in range(reps)) { - let tb = best_ms(1) { - matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) + w16 |> resize(n * d) + for (h, v in w16, w) { + h = bf16_bits(v) + } + let x <- race_fill(int(ntok * n), 2, 1.4, 0.0) + var y : array + y |> resize(ntok * d) + let family = get_bf16_batch_gen() + let min_n = get_f32_batch_tile_min_n() + let labels <- [family != null ? "bf16 plane, the stamped family" : "bf16 plane, the widen walk (no family stamped)", + "f32 plane, the tile walk", "bf16 plane, the widen walk", "bf16 plane, the row nest"] + var times <- time_rounds(length(labels), reps, 1, $(i : int) { + if (i == 0) { + set_bf16_batch_gen(family) + } else { + let none : Bf16BatchGenFn + set_bf16_batch_gen(none) } - bf = min(bf, tb) - let tf = best_ms(1) { + set_f32_batch_tile_min_n(i == 3 ? NEST_ONLY_MIN_N : min_n) + }, $(i : int) { + if (i == 1) { matmul_batch(y, w, x, n, d, ntok) + } else { + matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) } - ff = min(ff, tf) - } + }) + set_bf16_batch_gen(family) + set_f32_batch_tile_min_n(min_n) let fl = 2.0lf * double(n * d * ntok) / 1.0e6lf - print("bf16 n={n} d={d} ntok={ntok}: bf16 plane {bf} ms ({fl / bf} GFLOP/s, {bf16w_tileform_gen() > 0 ? "the stamped family" : "the widen walk"}), f32 plane {ff} ms ({fl / ff} GFLOP/s)\n") + for (lbl, us in labels, times) { + line("bf16 n={n} d={d} ntok={ntok}", lbl, us) $(ms : float) => "{fl / double(ms)} GFLOP/s" + } +} + +//! the GELU body before the float kernel: erfc as the NR rational fit over libm's exp, every op in double +def private gelu_erf_double(var x : array) { + for (v in x) { + let d = double(v) + let z = abs(d) * 0.7071067811865476lf + let t = 1.0lf / (1.0lf + 0.5lf * z) + let erfc = t * exp(-z * z - 1.26551223lf + t * (1.00002368lf + t * (0.37409196lf + t * (0.09678418lf + t * (-0.18628806lf + + t * (0.27886807lf + t * (-1.13520398lf + t * (1.48851587lf + t * (-0.82215223lf + t * 0.17087277lf))))))))) + v = float(d >= 0.0lf ? 0.5lf * d * (2.0lf - erfc) : 0.5lf * d * erfc) + } +} + +def private race_gelu(reps : int) { + let n = 1024 * 4096 + let x0 <- race_fill(n, 3, 8.0, 0.0) + var x : array + x |> resize(n) + let labels <- ["the float kernel", "the double body"] + var times <- time_rounds(length(labels), reps, 1, $(_i : int) { + for (d, s in x, x0) { + d = s + } + }, $(i : int) { + if (i == 0) { + gelu_erf(unsafe(addr(x[0])), int64(n)) + } else { + gelu_erf_double(x) + } + }) + for (lbl, us in labels, times) { + line("gelu_erf {n} elements", lbl, us) $(ms : float) => "{ms * 1.0e6 / float(n)} ns an element" + } } def private race_mel(reps : int) { @@ -93,21 +128,24 @@ def private race_mel(reps : int) { } set_single_thread_(true) var inscope a <- log_mel_qwen3a(samples, 128l) - let one = best_ms(reps) { - var inscope m <- log_mel_qwen3a(samples, 128l) - } set_single_thread_(false) var inscope b <- log_mel_qwen3a(samples, 128l) - let thr = best_ms(reps) { + let labels <- ["one lane", "the lanes"] + var times <- time_rounds(length(labels), reps, 1, $(i : int) { + set_single_thread_(i == 0) + }, $(_i : int) { var inscope m <- log_mel_qwen3a(samples, 128l) - } + }) + set_single_thread_(false) var diff = 0 for (p, q in a.data, b.data) { if (p != q) { diff++ } } - print("mel 60 s x 128: one lane {one} ms, the lanes {thr} ms, elements apart {diff} of {length(a.data)}\n") + for (lbl, us in labels, times) { + line("mel 60 s x 128", lbl, us) $(_ms : float) => "elements apart {diff} of {length(a.data)}" + } } let KP_L = 28l @@ -188,20 +226,18 @@ def private race_kpass(reps : int) { var att : array att |> resize(2l * KP_KVH * KP_SEQ) let mb = double(KP_L * KP_KEYS * kvdim * 2l) / 1.0e6lf - let h1 = best_ms(reps) { - kpass_heads(kall, q, att, 1) - } - let hn = best_ms(reps) { - kpass_heads(kall, q, att, int(KP_KVH * KP_NSL)) - } - let r1 = best_ms(reps) { - kpass_rows(kall, q, att, 1) + let labels <- ["(kv head, slice) units, one lane", "(kv head, slice) units, {KP_KVH * KP_NSL} units on the lanes", + "a slice over every kv head, one lane", "a slice over every kv head, {KP_NSL} units on the lanes"] + var times <- time_rounds(length(labels), reps, 1) $(i : int) { + if (i < 2) { + kpass_heads(kall, q, att, i == 0 ? 1 : int(KP_KVH * KP_NSL)) + } else { + kpass_rows(kall, q, att, i == 2 ? 1 : int(KP_NSL)) + } } - let rn = best_ms(reps) { - kpass_rows(kall, q, att, int(KP_NSL)) + for (lbl, us in labels, times) { + line("kpass {mb} MB of f16 keys a step", lbl, us) $(ms : float) => "{mb / double(ms)} GB/s, {ms / float(KP_L)} ms a layer" } - print("kpass {mb} MB of f16 keys a step, (kv head, slice) units: one lane {h1} ms ({mb / h1} GB/s), {KP_KVH * KP_NSL} units {hn} ms ({mb / hn} GB/s, {hn / double(KP_L)} ms a layer)\n") - print("kpass {mb} MB of f16 keys a step, a slice over every kv head: one lane {r1} ms ({mb / r1} GB/s), {KP_NSL} units {rn} ms ({mb / rn} GB/s, {rn / double(KP_L)} ms a layer)\n") } [export] @@ -223,6 +259,9 @@ def main() : int { race_bf16(4320l, 480l, 800l, cfg.reps) race_bf16(1152l, 1152l, 256l, cfg.reps) } + if (cfg.arm == "all" || cfg.arm == "gelu") { + race_gelu(cfg.reps) + } if (cfg.arm == "all" || cfg.arm == "mel") { race_mel(cfg.reps) } diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 35dcfafd54..8cbe12e849 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -943,10 +943,7 @@ struct Bf16wFixture { yref : array } -def private bf16_round(v : float) : float { - let bits = unsafe(reinterpret(v)) - return unsafe(reinterpret(((bits + 0x7FFFu + ((bits >> 16u) & 1u)) >> 16u) << 16u)) -} +def private bf16_round(v : float) : float => unsafe(reinterpret(uint(bf16_bits(v)) << 16u)) def build_bf16w_fixture(n, d, ntok : int64) : Bf16wFixture { var fx = Bf16wFixture(n = n, d = d, ntok = ntok) @@ -972,12 +969,11 @@ def build_bf16w_fixture(n, d, ntok : int64) : Bf16wFixture { return <- fx } -//! one variant over the fixture as bf16w_batch walks it, one lane; form 0 (the reference row) walks one group at four tokens +//! one variant over the fixture through the family's walk, one lane; form 0 (the reference row) walks one group at four tokens def run_bf16w_variant(tile : Bf16wTileFn; fx : Bf16wFixture; form, ts : int64; var y : array) { let groups = max(form, 1l) let step = form > 0l ? ts : 4l let ng = fx.d / 16l - let tbs = (fx.ntok / step) * step var panel : array panel |> resize(groups * 16l * fx.n) unsafe { @@ -987,23 +983,8 @@ def run_bf16w_variant(tile : Bf16wTileFn; fx : Bf16wFixture; form, ts : int64; v let xb = xbf16_build_f32(addr(fx.x[0]), fx.n, fx.ntok, xr) let xrc = reinterpret(xr) var pp = addr(panel[0]) - var g0 = 0l - while (g0 < ng) { - let gn = min(groups, ng - g0) - if (gn == groups) { - bf16w_panel_build(wp, fx.n, g0, groups, pp) - var tk = 0l - while (tk < tbs) { - invoke(tile, yp, reinterpret(pp), null, reinterpret(xb), null, null, fx.n, fx.d, g0, tk) - tk += step - } - if (tbs < fx.ntok) { - batch_tile_block(yp, wp, xrc, fx.n, fx.d, g0 * 16l, (g0 + groups) * 16l, tbs, fx.ntok, fx.ntok - tbs) - } - } else { - batch_tile_block(yp, wp, xrc, fx.n, fx.d, g0 * 16l, (g0 + gn) * 16l, 0l, fx.ntok, fx.ntok) - } - g0 += groups + bf16w_walk(yp, wp, xrc, fx.n, fx.d, 0l, ng, groups, fx.ntok, step, pp) $(g, tk : int64) { + invoke(tile, yp, reinterpret(pp), null, reinterpret(xb), null, null, fx.n, fx.d, g, tk) } if (ng * 16l < fx.d) { batch_tile_block(yp, wp, xrc, fx.n, fx.d, ng * 16l, fx.d, 0l, fx.ntok, fx.ntok) @@ -1011,18 +992,9 @@ def run_bf16w_variant(tile : Bf16wTileFn; fx : Bf16wFixture; form, ts : int64; v } } -//! `y` against the oracle at close_tol, widened to the bf16 envelope on a tile form: the TMUL sums its -//! pairs in another order than the double oracle, a few f32 ulps of the largest output over 2560 terms -def close_to_bf16w_ref(fx : Bf16wFixture; y : array; form : int64; var maxdiff : float&) : bool { - let envelope = bf16_envelope(fx.yref, length(y), form) - var ok = true - for (i in range64(fx.ntok * fx.d)) { - let diff = abs(y[i] - fx.yref[i]) - maxdiff = max(maxdiff, diff) - ok = ok && diff <= max(close_tol(fx.yref[i]), envelope) - } - return ok -} +//! the bar form of a bf16w row's compare: every row's tails run the box's f32 tile perm over 2560-term +//! dots that cancel, so the bf16 envelope holds the reference row as it holds a tile form +def private bf16w_bar_form(form : int64) : int64 => max(form, 1l) def bf16w_test_family(fxs : array) : bool { var tvs <- bf16w_tile_gen_variants() @@ -1043,9 +1015,9 @@ def bf16w_test_family(fxs : array) : bool { var inscope y : array y |> resize(fx.ntok * fx.d) run_bf16w_variant(v._1, fx, form, ts, y) - let fxok = close_to_bf16w_ref(fx, y, form, maxdiff) + let fxok = close_to_ref(fx, y, maxdiff, bf16w_bar_form(form)) if (!fxok) { - dump_mismatch("bf16w {v._0} n={fx.n}", y, fx.yref, int(fx.d)) $(i : int) => max(close_tol(fx.yref[i]), bf16_envelope(fx.yref, length(y), form)) + dump_mismatch("bf16w {v._0} n={fx.n}", y, fx.yref, int(fx.d)) $(i : int) => max(close_tol(fx.yref[i]), bf16_envelope(fx.yref, length(y), bf16w_bar_form(form))) } ok = ok && fxok } @@ -1087,7 +1059,7 @@ def bf16w_tune_family(var rows_out : table) : string { } var maxdiff = 0.0 run_bf16w_variant(v._1, fx, form, ts, y) - if (!close_to_bf16w_ref(fx, y, form, maxdiff)) { + if (!close_to_ref(fx, y, maxdiff, bf16w_bar_form(form))) { print("bf16w {v._0}: MISMATCH (maxdiff {maxdiff}) - aborting tune\n") best = "" break diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 03b94fde2d..8663957d0d 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -16,6 +16,8 @@ options _dasllama_internal = true require dasllama/dasllama_math // side-effect: loads the *_template kernels for the cross-module scan require dasllama/dasllama_math_default // dot_q8q8_template lives here +require dasllama/dasllama_math_gen // bf16_bits - the bf16 tile bench's weights round as the engine's planes do +require ../tests/_compares.das // erf_series - the GELU bench's oracle, the kernel cells' too require dasllama/dasllama_math_aarch64_neon // laneq4x4 template (the repack backend "arm64-gen" registers via dasllama_common's math_gen require) require dasllama/dasllama_kv_codec // nolint:STYLE030 — side-effect: the KV-codec *_template kernels for the cross-module scan require dasllama/dasllama_convert // side-effect: the f16 row-convert templates for the cross-module scan @@ -1018,33 +1020,11 @@ def bench_scale() : string { return report("scale_inplace", race) } -// ===== softmax: (var x, size) in-place — reset before each timing block; repeated softmax of a -// distribution stays a bounded distribution, so no denormal/inf drift over REPS. - -def bench_softmax() : string { - let vs <- softmax_template_variants() - var inscope d0, d : array - d0 |> resize(N) +//! an in-place `(var x, n)` kernel's race: `d0` reset before every run, the result gated f64-relative on `want` +def private bench_unary_inplace(kernel : string; vs : array>; d0 : array; want : array) : string { + var inscope d : array d |> resize(N) - for (i in range64(N)) { - d0[i] = 4.0f * sin(float(i) * 0.11f) // scores in [-4,4], a realistic logit spread - } var dp = unsafe(addr(d[0])) - var inscope want : array - want |> resize(N) - var mx = -1.0e30lf - for (i in range64(N)) { - mx = max(mx, double(d0[i])) - } - var wsum = 0.0lf - for (i in range64(N)) { - let e = exp(double(d0[i]) - mx) - want[i] = e - wsum += e - } - for (i in range64(N)) { - want[i] /= wsum - } var sink = 0.0f var inscope race <- race_variants(vs, $(i : int) : bool { reset_stream(dp, d0) @@ -1067,31 +1047,27 @@ def bench_softmax() : string { sink += d[0] }) tune_detail(" checksum {sink}\n") - return report("softmax", race) + return report(kernel, race) } -// ===== bf16_batch_tile: the f32 tile over a bf16 weight plane - the same gate, the weights bf16 bits. - -def private bf16_bits_of(v : float) : uint16 { - let bits = unsafe(reinterpret(v)) - return uint16((bits + 0x7FFFu + ((bits >> 16u) & 1u)) >> 16u) +//! the four-rows-by-four-tokens tile race over a weight plane of either lattice, gated f64-relative on all sixteen dots +def private w_val(wp : float const?; i : int64) : double => double(unsafe(wp[i])) +def private w_val(wp : uint16 const?; i : int64) : double { + unsafe { + return double(reinterpret(uint(wp[i]) << 16u)) + } } -def bench_bf16_batch_tile() : string { - let vs <- bf16_batch_tile_template_variants() - let D = 4l - var inscope w16 : array +def private bench_batch_tile(kernel : string; vs : array>; wp : auto const?) : string { + let D = 4l // y row stride: the tile is the whole y var inscope x, y : array - w16 |> resize(4l * N) var xp = aligned_f32(x, 4l * N) var yp = aligned_f32(y, 4l * D) unsafe { for (i in range64(4l * N)) { - w16[i] = bf16_bits_of(sin(float(i) * 0.1f)) xp[i] = cos(float(i) * 0.07f) } } - let wp = unsafe(addr(w16[0])) var sink = 0.0f var inscope race <- race_variants(vs, $(i : int) : bool { let f = vs[i]._1 @@ -1101,7 +1077,7 @@ def bench_bf16_batch_tile() : string { for (tk in range64(4l)) { var want = 0.0lf for (j in range64(N)) { - want += double(unsafe(reinterpret(uint(w16[r * N + j]) << 16u))) * double(unsafe(xp[tk * N + j])) + want += w_val(wp, r * N + j) * double(unsafe(xp[tk * N + j])) } let rel = rel_err(double(unsafe(yp[tk * D + r])), want) maxrel = worst_rel(maxrel, rel) @@ -1117,61 +1093,65 @@ def bench_bf16_batch_tile() : string { sink += y[0] }) tune_detail(" checksum {sink}\n") - return report("bf16_batch_tile", race) + return report(kernel, race) } -// ===== gelu_erf: in place over N, gated f64-relative against the erf series' GELU on a [-4, 4] spread. +// ===== softmax: (var x, size) in-place — reset before each timing block; repeated softmax of a +// distribution stays a bounded distribution, so no denormal/inf drift over REPS. + +def bench_softmax() : string { + let vs <- softmax_template_variants() + var inscope d0 : array + d0 |> resize(N) + for (i in range64(N)) { + d0[i] = 4.0f * sin(float(i) * 0.11f) // scores in [-4,4], a realistic logit spread + } + var inscope want : array + want |> resize(N) + var mx = -1.0e30lf + for (i in range64(N)) { + mx = max(mx, double(d0[i])) + } + var wsum = 0.0lf + for (i in range64(N)) { + let e = exp(double(d0[i]) - mx) + want[i] = e + wsum += e + } + for (i in range64(N)) { + want[i] /= wsum + } + return bench_unary_inplace("softmax", vs, d0, want) +} + +// ===== bf16_batch_tile: the f32 tile over a bf16 weight plane - the same gate, the weights bf16 bits. -def private erf_series(x : double) : double { - var sum = 0.0lf - var term = x - var k = 0 - while (abs(term) > 1.0e-18lf && k < 200) { - sum += term / double(2 * k + 1) - k++ - term *= -x * x / double(k) +def bench_bf16_batch_tile() : string { + let vs <- bf16_batch_tile_template_variants() + var inscope w16 : array + w16 |> resize(4l * N) + for (i in range64(4l * N)) { + w16[i] = bf16_bits(sin(float(i) * 0.1f)) } - return sum * 2.0lf / sqrt(3.141592653589793lf) + return bench_batch_tile("bf16_batch_tile", vs, unsafe(addr(w16[0]))) } +// ===== gelu_erf: in place over N, gated f64-relative against the erf series' GELU on a [-4, 4] spread. + def bench_gelu_erf() : string { let vs <- gelu_erf_template_variants() - var inscope d0, d : array + var inscope d0 : array d0 |> resize(N) - d |> resize(N) for (i in range64(N)) { d0[i] = 4.0f * sin(float(i) * 0.11f) } - var dp = unsafe(addr(d[0])) var inscope want : array want |> resize(N) for (i in range64(N)) { let x = double(d0[i]) want[i] = 0.5lf * x * (1.0lf + erf_series(x * 0.7071067811865476lf)) } - var sink = 0.0f - var inscope race <- race_variants(vs, $(i : int) : bool { - reset_stream(dp, d0) - let f = vs[i]._1 - invoke(f, dp, N) - var maxrel = 0.0lf - for (j in range64(N)) { - let rel = rel_err(double(d[j]), want[j]) - maxrel = worst_rel(maxrel, rel) - } - return maxrel <= TOL - }, $(_i : int) { - reset_stream(dp, d0) - }, $(i : int) { - let f = vs[i]._1 - for (_rep in range(reps_of(i))) { - invoke(f, dp, N) - } - }, $ { - sink += d[0] - }) - tune_detail(" checksum {sink}\n") - return report("gelu_erf", race) + return bench_unary_inplace("gelu_erf", vs, d0, want) } // ===== rmsnorm: (var o, x, weight, size, eps) out-of-place — repeatable, no reset needed. @@ -1817,43 +1797,14 @@ def bench_gemm_tile() : string { def bench_f32_batch_tile() : string { let vs <- f32_batch_tile_template_variants() - let D = 4l // y row stride: the tile is the whole y - var inscope w, x, y : array + var inscope w : array var wp = aligned_f32(w, 4l * N) - var xp = aligned_f32(x, 4l * N) - var yp = aligned_f32(y, 4l * D) unsafe { for (i in range64(4l * N)) { wp[i] = sin(float(i) * 0.1f) - xp[i] = cos(float(i) * 0.07f) } + return bench_batch_tile("f32_batch_tile", vs, reinterpret(wp)) } - var sink = 0.0f - var inscope race <- race_variants(vs, $(i : int) : bool { - let f = vs[i]._1 - invoke(f, yp, wp, xp, N, D) - var maxrel = 0.0lf - for (r in range64(4l)) { - for (tk in range64(4l)) { - var want = 0.0lf - for (j in range64(N)) { - want += double(unsafe(wp[r * N + j])) * double(unsafe(xp[tk * N + j])) - } - let rel = rel_err(double(unsafe(yp[tk * D + r])), want) - maxrel = worst_rel(maxrel, rel) - } - } - return maxrel <= TOL - }, $(_i : int) {}, $(i : int) { - let f = vs[i]._1 - for (_rep in range(reps_of(i))) { - invoke(f, yp, wp, xp, N, D) - } - }, $ { - sink += y[0] - }) - tune_detail(" checksum {sink}\n") - return report("f32_batch_tile", race) } // ===== dot_q8q8_laneq4x4: arm64+JIT only (the repack backend must be registered so the pin lands diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index dd95490392..8ede3da40a 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -1009,7 +1009,7 @@ compact rows on sixteen windows (one ragged) against `attention_bidir_windows` w over the same rows as the leak control, the rms seam and the gated hidden (the shared bias-gate stamp `TowerBiasGate16`, the qwen25v hidden's) against their CPU forms; the bias class's erf arm (the whisper-class towers' GELU) against `gelu_erf_batch` at 1e-5 -relative - the f32 evaluation of the CPU's double erfc - with the tanh arm missing that bar as the +relative - the shader's f32 evaluation of the CPU kernel's erfc fit - with the tanh arm missing that bar as the told-apart control, and its silu arm (canary's FFN) against `silu` with the tanh arm as its control; the Conformer classes (the silu rows - the bias class's silu arm with no bias row - the GLU, the causal depthwise taps, the chunk-12 relative-position attention on both head-width stamps, @@ -2870,7 +2870,8 @@ Q8_0 blocks where the device feeds f16), the tail counter `vulkan_tower_tail_enc alone with the blocks on the device on both, another mel's rows outside the bar as the control, all of which skip without their carriers (the qwen2audio / voxtral / omni-3b f32 mmprojs, the E2B bf16 mmproj, the canary f32 encoder, the parakeet f32 encoder, the ultravox f16 mmproj, the Qwen3-ASR bf16 mmproj), without jfk.wav, without a Vulkan device under `DASLLAMA_GPU=1`, and on a das_metal build. -`test_whisper.das` - stocked suite; model-gated: the whisper/parakeet/canary/gemma4a/omni +`test_whisper.das` - stocked suite; the qwen3a mel cell (model-free: the lanes' mel image against the one-lane +image element for element - the DFT GEMM splits on whole 4-row tiles so every row rounds as one lane's); model-gated: the whisper/parakeet/canary/gemma4a/omni oracle cells (the parakeet v2 cell also runs the transcription with single-thread mode off and on: the team leg dispatches, the one-lane leg never does, and its tokens are the team leg's - the decode step's one-lane form against its team publish, both texts logged - and the session that diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index bae822c8ef..f5fa9947da 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -34,10 +34,11 @@ through a `../dasllama/` function and calls no other `../dasllama/` function. changes which `run.das` suites list one, applies that checklist too.** **A diff that adds a cell or an assert whose expected value must be kept in step with something -maintained outside the cell (a document, a checked-in table, a committed artifact's form, a roster -or a knob list) - never a value the cell's own claim defines - or that a checked-in table names as -its evidence, applies `REVIEW_PINNED_GATES.md` too.** An assert that compares against a -`../dasllama/` constant by name, not against a literal copy of it, keeps nothing in step. +maintained outside the cell - anything the cell restates as a literal or a copied condition: a +document, a checked-in table, a committed artifact's form, a roster, a knob list, an engine dispatch +condition - never a value the cell's own claim defines - or that a checked-in table names as its +evidence, applies `REVIEW_PINNED_GATES.md` too.** An assert that compares against a `../dasllama/` +constant by name, not against a literal copy of it, keeps nothing in step. **A cell that calls - in its own code or through a helper - a `../dasllama/` call that sets a module or process global a later load, route choice or kernel dispatch reads (a `set_*`, @@ -199,15 +200,16 @@ number can produce.** A value set through a knob setter or a config field (`set_ cap, a resize, or a counter showing a route ran is no such count, and a device's geometry (subgroup width, SM count) is no coverage claim. -**An exact token or id compare over a prompt whose continuation can tie states in the cell what -makes its two sides compute bit-identical logits - one shared entry point, an assert pinning the -lane, or a lane pin (a call that picks the weight form, the Metal tensor crowns or the CPU kernel -backend a load serves on) plus why the one storage layout (a flat K/V cache against a paged one) -or batch shape that alone differs leaves the logits bit-identical; a compare no such statement -covers - different lanes, backends or kernel forms among them - is a defect, and takes the -forced-feed logits-tolerance form: the same fixed tokens fed to both sides, the logits compared -within a bar.** A continuation can tie unless every next token is dictated by the prompt's own -sequence (a count, an arithmetic series, a fixed enumeration); prose always can. +**An exact compare of a text model's tokens or ids over a prompt whose continuation can tie states +in the cell what makes its two sides compute bit-identical logits - one shared entry point, an +assert pinning the lane, or a lane pin (a call that picks the weight form, the Metal tensor crowns +or the CPU kernel backend a load serves on) plus why the one storage layout (a flat K/V cache +against a paged one) or batch shape that alone differs leaves the logits bit-identical; a compare no +such statement covers - different lanes, backends or kernel forms among them - is a defect, and +takes the forced-feed logits-tolerance form: the same fixed tokens fed to both sides, the logits +compared within a bar.** A continuation can tie unless every next token is dictated by the prompt's +own sequence (a count, an arithmetic series, a fixed enumeration); prose always can. An ASR +transcript compare answers to `REVIEW_MEDIA_CELLS.md`. **A test in this folder that loads a stocked artifact whose producer none of the following names is a defect.** Any one of these names the producer: a row in `../performance/model_specs.das`; a diff --git a/modules/dasLLAMA/tests/_compares.das b/modules/dasLLAMA/tests/_compares.das index 09a3e8f4c0..4d77444fd2 100644 --- a/modules/dasLLAMA/tests/_compares.das +++ b/modules/dasLLAMA/tests/_compares.das @@ -594,6 +594,19 @@ def sweep_inert(var r : Sweep; got, before : array) { } } +//! erf by its Maclaurin series in double, run until a term is under 1e-18 - the GELU oracles' erf +def erf_series(x : double) : double { + var sum = 0.0lf + var term = x + var k = 0 + while (abs(term) > 1.0e-18lf && k < 200) { + sum += term / double(2 * k + 1) + k++ + term *= -x * x / double(k) + } + return sum * 2.0lf / sqrt(3.141592653589793lf) +} + //! The sweep's verdict: `cases` lengths run (0 = at least one), none off its bar, no poison leg inside it, the largest //! difference logged; `moved` also asks every element of an in-place run to have moved. def report_sweep(t : T?; name : string; r : Sweep; cases : int = 0; moved : bool = false) { diff --git a/modules/dasLLAMA/tests/test_matmul_batch.das b/modules/dasLLAMA/tests/test_matmul_batch.das index f84ee92b2f..109514cf4b 100644 --- a/modules/dasLLAMA/tests/test_matmul_batch.das +++ b/modules/dasLLAMA/tests/test_matmul_batch.das @@ -199,12 +199,12 @@ def private bf16_bits_of(v : float) : uint16 { //! the bf16 plane through the same walk: Y against the plain oracle over the widened weights within //! the tile's rounding, a NaN sentinel under every output, the poisoned expectation the control; -//! where the box stamps the half-width family (`bf16w_tileform_gen() > 0`, every call of ntok past a -//! tile on an n of whole 32s) the oracle's activations are the family's bf16-rounded ones +//! where the stamped half-width family serves the shape (`bf16w_batch_serves`) the oracle's +//! activations are the family's bf16-rounded ones def check_bf16(t : T?; n, d, ntok : int64) { let wf <- fillf(n * d) var x <- fillf(ntok * n) - if (bf16w_tileform_gen() > 0 && n % 32l == 0l && ntok > 1l && ntok >= int64(bf16w_tokstep_gen())) { + if (ntok > 1l && bf16w_batch_serves(n, d, ntok)) { for (k in range64(ntok * n)) { x[k] = unsafe(reinterpret(uint(bf16_bits_of(x[k])) << 16u)) } @@ -223,6 +223,7 @@ def check_bf16(t : T?; n, d, ntok : int64) { matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) } t |> success(all_finite(y), "bf16 batch {n}x{d} ntok={ntok}: every output finite") + t |> success(all_finite(want), "bf16 batch {n}x{d} ntok={ntok}: every oracle element finite") t |> equal(mismatch_rel("bf16 batch {n}x{d} ntok={ntok}", y, want, F32_TILE_REL), 0, "bf16 batch {n}x{d} ntok={ntok}: elements past {F32_TILE_REL} relative of the plain oracle over the widened weights") want[(ntok * d) / 2l] += 1.0e-3 * max(abs(want[(ntok * d) / 2l]), 1.0) t |> success(mismatch_rel("bf16 batch {n}x{d} ntok={ntok} poisoned", y, want, F32_TILE_REL) > 0, "control: the poisoned element must miss the {F32_TILE_REL} bar") diff --git a/modules/dasLLAMA/tests/test_metal_misc_kernels.das b/modules/dasLLAMA/tests/test_metal_misc_kernels.das index 1cf3a20125..72ec09a3c9 100644 --- a/modules/dasLLAMA/tests/test_metal_misc_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_misc_kernels.das @@ -942,7 +942,7 @@ def private gelu_flavor_gate(t : T?; var dev, queue; total : int; lut : bool) { } return if (!ran || total < 8) // LUT: one f16 ulp at |g|~10 (RNE tie flips between the tanh and exp-identity evals); - // erf: the CPU's double-precision NR erfc against the same form in f32 + // erf: the CPU's f32 NR erfc fit against the shader's evaluation of the same fit - a few ulp apart check_buf(t, "{tag}: form-matched inside the flavor bar", mc.buf[bg], want, Bar(abs_floor = lut ? 1e-2 : 1e-5), FLAT_SHAPE) var inscope back <- mc |> plane_run(bg) t |> success(back[total - 1] == 50.0 && back[total - 3] == 100.0, "{tag}: positive saturation exact") diff --git a/modules/dasLLAMA/tests/test_model_image.das b/modules/dasLLAMA/tests/test_model_image.das index 62435471fc..01bfc3d36b 100644 --- a/modules/dasLLAMA/tests/test_model_image.das +++ b/modules/dasLLAMA/tests/test_model_image.das @@ -2434,6 +2434,21 @@ def test_gemma4a_encoder_image_roundtrip(t : T?) { } } +//! the q8 tower's transcript against the f32 rail's as word lists - a comma rides a near-tie token +//! the two rails' rounding can split - both lists logged, the control a list one word short +def same_words(t : T?; who : string; q8_text, f32_text : string) { + let q8 = join(transcript_words(q8_text), " ") + let f32 = join(transcript_words(f32_text), " ") + to_log(LOG_INFO, "{who} q8 words : {q8}\n") + to_log(LOG_INFO, "{who} f32 words: {f32}\n") + t |> equal(q8, f32, "q8 tower transcript matches the f32 rail, punctuation aside") + var short <- transcript_words(q8_text) + if (!empty(short)) { + short |> erase(length(short) - 1) + } + t |> success(join(short, " ") != f32, "control: the q8 words one short miss the f32 rail") +} + [test] def test_gemma4a_transcribe_q8_ab(t : T?) { t |> run("gemma4a transcribe: the q8 tower vs the f32 rail on jfk, the same words") @(t : T?) { @@ -2469,10 +2484,7 @@ def test_gemma4a_transcribe_q8_ab(t : T?) { reset_gemma4a_q8() delete dec } - to_log(LOG_INFO, "gemma4a q8 : {q8_text}\n") - to_log(LOG_INFO, "gemma4a f32: {f32_text}\n") - // the words, not the punctuation: a comma rides a near-tie token the two rails' rounding can split - t |> equal(join(transcript_words(q8_text), " "), join(transcript_words(f32_text), " "), "q8 tower transcript matches the f32 rail, punctuation aside") + same_words(t, "gemma4a", q8_text, f32_text) delete samples } } @@ -2734,10 +2746,7 @@ def test_qwen3a_transcribe_q8_ab(t : T?) { reset_qwen3a_q8() delete dec } - to_log(LOG_INFO, "qwen3a q8 : {q8_text}\n") - to_log(LOG_INFO, "qwen3a f32: {f32_text}\n") - // the words, not the punctuation: a comma rides a near-tie token the two rails' rounding can split - t |> equal(join(transcript_words(q8_text), " "), join(transcript_words(f32_text), " "), "q8 tower transcript matches the f32 rail, punctuation aside") + same_words(t, "qwen3a", q8_text, f32_text) delete samples } } diff --git a/modules/dasLLAMA/tests/test_tower_asr_kernels.das b/modules/dasLLAMA/tests/test_tower_asr_kernels.das index 193421df8d..764b04f328 100644 --- a/modules/dasLLAMA/tests/test_tower_asr_kernels.das +++ b/modules/dasLLAMA/tests/test_tower_asr_kernels.das @@ -311,16 +311,6 @@ let GE_POS = 25l let GE_REL = 1.0e-6lf let GE_FLOOR = 1.0e-6lf -def private erf_ref(z : double) : double { - var term = z - var sum = z - for (k in range(1, 60)) { - term *= -z * z / double(k) - sum += term / double(2 * k + 1) - } - return sum * 2.0lf / sqrt(DBL_PI) -} - def private gelu_fixture(var x : array) { lcg_seed(0x9E10ul) fill(x, GE_D * GE_POS, 4.0) @@ -331,7 +321,7 @@ def private gelu_ref(x : array; var want : array; var env : array env |> resize(long_length(x)) for (v, w, e in x, want, env) { let d = double(v) - w = 0.5lf * d * (1.0lf + erf_ref(d * 0.7071067811865476lf)) + w = 0.5lf * d * (1.0lf + erf_series(d * 0.7071067811865476lf)) e = abs(d) } } diff --git a/modules/dasLLAMA/tests/test_vulkan_tower_kernels.das b/modules/dasLLAMA/tests/test_vulkan_tower_kernels.das index 50b14cbd96..c358a52432 100644 --- a/modules/dasLLAMA/tests/test_vulkan_tower_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_tower_kernels.das @@ -1120,9 +1120,9 @@ def test_vkt_tower_ln_rows(t0 : T?) { // nolint:STYLE038 — one flat gate ove check_rows(t, "tower_bias_act BIAS_ACT_RELU the bias and relu vs max(x + b, 0)", grelu_dev, grelu_ref, BAR_APPROX, d) gelu_tanh_lut_batch(g_ref, int64(d), int64(rows)) check_rows(t, "tower_bias_act the bias and the tanh GELU vs add_bias_rows + gelu_tanh_lut_batch", g_dev, g_ref, BAR_APPROX, d) - // the erf arm evaluates the CPU's rational erfc in f32 where the CPU evaluates it in double: 1e-5 relative holds the - // f32 rounding of the polynomial and the exp (a few ulp) and is well under the erf-vs-tanh gap (4.7e-4 at x = -2.7), - // which the approx bar (2e-3) could not tell apart + // the erf arm evaluates the CPU's rational erfc fit in the shader's f32 as the CPU kernel does on its lattice: 1e-5 + // relative holds the two evaluations' rounding of the polynomial and the exp (a few ulp) and is well under the + // erf-vs-tanh gap (4.7e-4 at x = -2.7), which the approx bar (2e-3) could not tell apart gelu_erf_batch(gerf_ref, int64(d), int64(rows)) let erf_bar = 1e-5 check_rows(t, "tower_bias_act BIAS_ACT_GELU_ERF the bias and the erf GELU vs add_bias_rows + gelu_erf_batch within {erf_bar}", gerf_dev, gerf_ref, Bar(rel = erf_bar, poison = 0.5), d) diff --git a/modules/dasLLAMA/tests/test_whisper.das b/modules/dasLLAMA/tests/test_whisper.das index d900e510f4..80df982826 100644 --- a/modules/dasLLAMA/tests/test_whisper.das +++ b/modules/dasLLAMA/tests/test_whisper.das @@ -295,6 +295,32 @@ def test_qwen3a_tower(t : T?) { } } +[test] +def test_qwen3a_mel_lanes(t : T?) { + t |> run("qwen3a mel: the lanes' image is the one-lane image, element for element") @(t : T?) { + var samples : array + samples |> resize(16000 * 10) + for (i in range(16000 * 10)) { + samples[i] = 0.3 * sin(float(i) * 0.01) + 0.1 * sin(float(i) * 0.37) + } + with_dasllama_jobque_() { + set_single_thread_(true) + var inscope one <- log_mel_qwen3a(samples, 128l) + set_single_thread_(false) + var inscope lanes <- log_mel_qwen3a(samples, 128l) + t |> equal(one.n_len, lanes.n_len, "the two images span the same frames") + t |> success(!empty(one.data), "the image has elements") + var apart = 0 + for (p, q in one.data, lanes.data) { + if (p != q) { + apart++ + } + } + t |> equal(apart, 0, "elements apart between the one-lane and the lanes' mel") + } + } +} + [test] def test_parakeet_oracle(t : T?) { t |> run("parakeet-TDT greedy on jfk.wav is token-for-token with parakeet-cli") @(t : T?) { From c7865482284f6b6d4f63fc0ebb4a8e988b0ba2f2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 22:08:15 -0700 Subject: [PATCH 17/18] dasLLAMA: the CPU ASR race on the benchmarks checklist, the class seats The race script names its baselines and compare regions, holds every compared arm to a reference or to its baseline with the bound on the row, prints the tune manifest on every row, drops one warmup round and says so, runs the mel as an attribution sweep with timing-only rows, and exits non-zero when it reports nothing; the shared fill reserves before it sizes. The probe's bf16w tune rounds run in turn across the candidates, its rows name the envelope. The six class profiles carry the bf16w seat - the Granite Rapids mint's AMX perm on x86-amx, the one candidate elsewhere - so a fat release ships the family as a profile clone. The qwen3a mel is sanctioned uncovered in the hot-path coverage model, as the tokenizer is; the gates, hot-path, memory and placement checklists carry their self-review batches; the math_gen charter names the two new operand builders; rows 240 and 242 tag their -jit figures and cite the board. Co-Authored-By: Claude Fable 5.1 --- .../dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md | 86 +++--- modules/dasLLAMA/ARCHITECTURE_RUNTIME.md | 4 +- modules/dasLLAMA/REVIEW_GATES.md | 3 +- modules/dasLLAMA/REVIEW_HOT_PATH.md | 28 +- modules/dasLLAMA/REVIEW_MEMORY.md | 27 +- modules/dasLLAMA/REVIEW_PLACEMENT.md | 38 +-- modules/dasLLAMA/followup_general.md | 38 +-- modules/dasLLAMA/harness/_race_fixture.das | 1 + modules/dasLLAMA/harness/cpu_asr_race.das | 270 +++++++++++++----- modules/dasLLAMA/harness/gen_tune_probe.das | 65 +++-- modules/dasLLAMA/harness/tune_kernels.das | 7 +- .../defaults/arm-i8mm.tune-defaults.json | 1 + .../defaults/arm-neon.tune-defaults.json | 1 + .../defaults/arm-sve.tune-defaults.json | 1 + .../defaults/x86-amx.tune-defaults.json | 1 + .../defaults/x86-avx2.tune-defaults.json | 1 + .../defaults/x86-vnni512.tune-defaults.json | 1 + 17 files changed, 370 insertions(+), 203 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md index 0223a54243..7e8d3ce1bd 100644 --- a/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md +++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md @@ -245,47 +245,47 @@ repack walks that plane's region alone (`shexp_plane_q8`). ### CPU kernel tiers {#cpu-kernel-tiers} - **`dasllama_math.das`** - the numeric ABSTRACTION: typedefs, active backend pointers, public - wrappers, dispatch shaping, and the array-sizing helpers every tier and the engine share - (`reserve_resize`, `grow_resize`, `ensure_length`, `overwrite_resize`), the `[tuned]` fp32 - loop-hint families (`dot`, `axpy`, the elementwise set, `f32_batch_tile`) and the fp32 batch walk - (`matmul_batch_core`, `batch_tile_block`, generic over the f32 and bf16 weight lattices - `bf16_batch_tile` - widens each halfword by the exact shift); every other kernel lives in a tier file. +wrappers, dispatch shaping, and the array-sizing helpers every tier and the engine share +(`reserve_resize`, `grow_resize`, `ensure_length`, `overwrite_resize`), the `[tuned]` fp32 loop-hint +families (`dot`, `axpy`, the elementwise set, `f32_batch_tile`) and the fp32 batch walk +(`matmul_batch_core`, `batch_tile_block`, generic over the f32 and bf16 weight lattices - +`bf16_batch_tile` widens each halfword by the exact shift); every other kernel lives in a tier file. - **`dasllama_math_default.das`** - the portable backend, always registered, always correct, - out-ranked by any platform tier - the per-32 rail's portable kernels and scalar leaves, which - seat every backend's unset per-32 slot (`#per-32-rail`) - and the ONE body of every Q8*Q8 and - mx4 kernel shape: each shape is a `def template` over a dot and a chunk-split placeholder, and `[from_template]` stamps - it per (dot, split) and per weight-scale plane (the f32 plane and the wscale_f16 halfword plane, - bit-identical over the widened plane). The idot4 stamps the arm64 tier registers live here too: - the `idot4` builtin is plain das that every target lowers for itself. -- **`dasllama_math_aarch64_neon.das`** - the arm64 lane-indexed SDOT tier: the `sdot4_laneq` / - `tbl16` composers only, plus the registration of the idot4 stamps as "arm64-sdot". Its `[init]` - never fires off-arch (`ARCHITECTURE_INVARIANTS.md#inherited-invariants`, three-layer safety model), so an - intrinsic here needs a correct scalar fallback body, not a guard at the call site. -- **`dasllama_math_accelerate.das`** - the Accelerate/BNNS float tier (AMX on M1-M3, SME on M4+), - for genuinely-float planes only. BLAS-for-quant is ruled out structurally - (`ARCHITECTURE_INVARIANTS.md#inherited-invariants`). -- **`dasllama_math_gen.das`** / **`dasllama_gemm_gen.das`** / **`dasllama_gemm_schema.das`** / - **`dasllama_gemm_register.das`** - the generated GEMM tier: the runtime registration, the tile - generator, the layout/perm schema shared by generator and runtime, and the `[tune]` family - registration. A hand-written tile that the generator could emit belongs in the generator. - The bf16 tile's operand builders - the activation widen (`xbf16_build`, its `bf16_bits` - narrow) and the row-dequant reference panel (`kq_panel_rows_bf16`) - sit in - `dasllama_math_gen.das` beside the walk that runs them once per batch call, in its JIT - partition (`ARCHITECTURE_CPU_KERNELS.md#jit-partition-inlining`). The JIT cache keys a - generated kernel by its stub's hash and its perm's arguments, never by the generator's text, - so `REVIEW.das`'s `check_gemm_gen_stamp` pins one hash over `dasllama_gemm_gen.das` and - `dasllama_gemm_schema.das` with their comments and blank lines cut: a code edit to either is red until the stamp - moves, and the finding asks for the `LLVM_JIT_CODEGEN_VERSION` bump that re-keys the cached - kernels. The check licenses no names. -- **`dasllama_tune.das`** - the per-box loop-hint tuner (`[tuned]` / `[dasllama_grid]`) and the - perm-less clone `[from_template]` (a template body into an empty stub, placeholder calls - renamed to the annotation's targets - the kernel shapes' stamp). Tuning - POLICY lives here; tuned VALUES live in the box's sidecar, never in source. The framework is - OPTIONAL: `llvm_tune` comes through the `tune_framework` group, empty in a build that did not - configure dasLLVM, and every use of it sits behind `static_if (typeinfo module_exists(llvm_tune))`, so a build without dasLLVM opens no - sidecar, reads no policy env and reports nothing - `[tuned]` stamps the declared `fallback=` - (or `DEFAULT_PERM`) and the AST rewrite is unchanged. `box_profile_verdict_at` answers the - reason NAME rather than llvm_tune's enum for the same reason: a signature cannot be gated. -- **`dasllama_tune_scope.das`** - the one `[tune_scope]` declaration (tuner, covered modules, - version pin, shipped defaults), in a module every kernel module requires: a kernel reads its - class entry in the defaults profile at its own compile, so the scope must be on the AST first. +out-ranked by any platform tier - the per-32 rail's portable kernels and scalar leaves, which seat +every backend's unset per-32 slot (`#per-32-rail`) - and the ONE body of every Q8*Q8 and mx4 kernel +shape: each shape is a `def template` over a dot and a chunk-split placeholder, and +`[from_template]` stamps it per (dot, split) and per weight-scale plane (the f32 plane and the +wscale_f16 halfword plane, bit-identical over the widened plane). The idot4 stamps the arm64 tier +registers live here too: the `idot4` builtin is plain das that every target lowers for itself. - +**`dasllama_math_aarch64_neon.das`** - the arm64 lane-indexed SDOT tier: the `sdot4_laneq` / `tbl16` +composers only, plus the registration of the idot4 stamps as "arm64-sdot". Its `[init]` never fires +off-arch (`ARCHITECTURE_INVARIANTS.md#inherited-invariants`, three-layer safety model), so an +intrinsic here needs a correct scalar fallback body, not a guard at the call site. - +**`dasllama_math_accelerate.das`** - the Accelerate/BNNS float tier (AMX on M1-M3, SME on M4+), for +genuinely-float planes only. BLAS-for-quant is ruled out structurally +(`ARCHITECTURE_INVARIANTS.md#inherited-invariants`). - **`dasllama_math_gen.das`** / +**`dasllama_gemm_gen.das`** / **`dasllama_gemm_schema.das`** / **`dasllama_gemm_register.das`** - +the generated GEMM tier: the runtime registration, the tile generator, the layout/perm schema shared +by generator and runtime, and the `[tune]` family registration. A hand-written tile that the +generator could emit belongs in the generator. The bf16 tile's operand builders - the activation +widen (`xbf16_build`, its `bf16_bits` narrow, the f32 plane's `xbf16_build_f32`), the row-dequant +reference panel (`kq_panel_rows_bf16`) and the native bf16 plane's k-pair panel +(`bf16w_panel_build`) - sit in `dasllama_math_gen.das` beside the walk that runs them once per batch +call, in its JIT partition (`ARCHITECTURE_CPU_KERNELS.md#jit-partition-inlining`). The JIT cache +keys a generated kernel by its stub's hash and its perm's arguments, never by the generator's text, +so `REVIEW.das`'s `check_gemm_gen_stamp` pins one hash over `dasllama_gemm_gen.das` and +`dasllama_gemm_schema.das` with their comments and blank lines cut: a code edit to either is red +until the stamp moves, and the finding asks for the `LLVM_JIT_CODEGEN_VERSION` bump that re-keys the +cached kernels. The check licenses no names. - **`dasllama_tune.das`** - the per-box loop-hint tuner +(`[tuned]` / `[dasllama_grid]`) and the perm-less clone `[from_template]` (a template body into an +empty stub, placeholder calls renamed to the annotation's targets - the kernel shapes' stamp). +Tuning POLICY lives here; tuned VALUES live in the box's sidecar, never in source. The framework is +OPTIONAL: `llvm_tune` comes through the `tune_framework` group, empty in a build that did not +configure dasLLVM, and every use of it sits behind `static_if (typeinfo module_exists(llvm_tune))`, +so a build without dasLLVM opens no sidecar, reads no policy env and reports nothing - `[tuned]` +stamps the declared `fallback=` (or `DEFAULT_PERM`) and the AST rewrite is unchanged. +`box_profile_verdict_at` answers the reason NAME rather than llvm_tune's enum for the same reason: a +signature cannot be gated. - **`dasllama_tune_scope.das`** - the one `[tune_scope]` declaration +(tuner, covered modules, version pin, shipped defaults), in a module every kernel module requires: a +kernel reads its class entry in the defaults profile at its own compile, so the scope must be on the +AST first. \ No newline at end of file diff --git a/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md b/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md index f881cf1a98..e8ab6c9a99 100644 --- a/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md +++ b/modules/dasLLAMA/ARCHITECTURE_RUNTIME.md @@ -172,7 +172,9 @@ and stop-string tails a tick by design, so an entry contract would name its own instrument (`harness/served_bench.das`), and its one logging per-token branch - the budget's forced close - is a `[cold_path]` helper. The batched step's `eval_batch_` (reached through `scheduler_step` and the public `eval_batch`) and the media prefill's `eval_embd_` are uncovered the same way: each grows its workspace and its row set a call by design, and their perf gates are -the batched-row cells and the served-turn instrument. +the batched-row cells and the served-turn instrument. The qwen3a log-mel (`log_mel_qwen3a`) is uncovered the same +way: it builds its window, twiddles, filterbank and image a clip by design, one call a transcription, and its perf gate +is the ASR cells' encode wall. ### The post-CPU-burn GPU ramp and the residency heartbeat {#the-post-cpu-burn-gpu-ramp-and} diff --git a/modules/dasLLAMA/REVIEW_GATES.md b/modules/dasLLAMA/REVIEW_GATES.md index 204bb63456..82cb78faf7 100644 --- a/modules/dasLLAMA/REVIEW_GATES.md +++ b/modules/dasLLAMA/REVIEW_GATES.md @@ -6,7 +6,8 @@ doc: `ARCHITECTURE.md`. Planned work: `followup_general.md`. **An edit to a `REVIEW.das` under this folder never weakens a check: drops it while code it guards remains in the tree and no lint takes it over, narrows what it walks (a file or folder it stops reading), edits the fixed value a check compares against - a hash, a count or a list - -where the finding text does not tell the author to, or leaves it passing code its finding text +where the finding text does not tell the author to or the condition it states for doing so +does not hold, or leaves it passing code its finding text still names as a defect - fix the flagged code instead.** **A finding text names what failed; a diff that rewords one keeps it naming what failed.** diff --git a/modules/dasLLAMA/REVIEW_HOT_PATH.md b/modules/dasLLAMA/REVIEW_HOT_PATH.md index 0e0b9f4b89..fa9019dca3 100644 --- a/modules/dasLLAMA/REVIEW_HOT_PATH.md +++ b/modules/dasLLAMA/REVIEW_HOT_PATH.md @@ -3,20 +3,20 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE_RUNTIME.md`. Planned work: `followup_general.md`. -A function calls another when its body names that function in a call, a call written inside a -block or lambda literal included; the function that encloses the literal is the caller. A -function reaches another when it calls it, or calls a function that reaches it. A serving step -is one unit of work the runtime repeats inside one request it serves: a token, a prefill quantum -(one batch of prompt tokens the prefill path processes in a single pass), one encoded media input -of that request (an image, a video frame, an audio chunk), or one synthesized speech chunk or -frame. A region entry is the outermost function the runtime re-enters once per serving step, and -any function a serving -step runs through a function value - one a seat (a module-level variable holding a function, -filled through a public setter) or a kernel-backend override registration stores, after its -declaration, for the runtime to call through - where no `[hot_path]`, contract or `[cold_path]` -function reaches the `invoke`. An interior function is a function, not itself a region entry, -that has at least one caller under `dasllama/` and whose every such caller the runtime re-enters -once per serving step (`ARCHITECTURE_RUNTIME.md#the-hot-path-coverage-model`). +A function calls another when its body names that function in a call, a call written inside a block +or lambda literal included; the function that encloses the literal is the caller; an `invoke` +through a seat or a function value names no function and calls none. A function reaches another when +it calls it, or calls a function that reaches it. A serving step is one unit of work the runtime +repeats inside one request it serves: a token, a prefill quantum (one batch of prompt tokens the +prefill path processes in a single pass), one encoded media input of that request (an image, a video +frame, an audio chunk), or one synthesized speech chunk or frame. A region entry is the outermost +function the runtime re-enters once per serving step, and any function a serving step runs through a +function value - one a seat (a module-level variable holding a function, filled through a public +setter) or a kernel-backend override registration stores, after its declaration, for the runtime to +call through - where no `[hot_path]`, contract or `[cold_path]` function reaches the `invoke`. An +interior function is a function, not itself a region entry, that has at least one caller under +`dasllama/` and whose every such caller the runtime re-enters once per serving step +(`ARCHITECTURE_RUNTIME.md#the-hot-path-coverage-model`). **A diff that adds a kernel dispatch, a loop or a call that the runtime re-enters once per serving step makes sure the function that records the dispatch or holds the loop or call is, or is reached diff --git a/modules/dasLLAMA/REVIEW_MEMORY.md b/modules/dasLLAMA/REVIEW_MEMORY.md index c74abc9038..bee2929e59 100644 --- a/modules/dasLLAMA/REVIEW_MEMORY.md +++ b/modules/dasLLAMA/REVIEW_MEMORY.md @@ -25,30 +25,27 @@ body (`dasllama/dasllama_par.das`), which can dispatch onto those same lanes. **A module global whose declaration initializer holds a function value - `var g_x = @@fn`, or a table or struct literal with a `@@fn` field - is a defect unless an `[init]` in its file sets it -again when it reads null; a `let` holding a function value is a defect.** A serialized exe - one -that restores its built program, globals included, from a saved image at startup - and a forked +again when it reads null; a module `let` holding a function value is a defect.** A serialized exe - +one that restores its built program, globals included, from a saved image at startup - and a forked context restore globals as data, so a declaration initializer alone arrives null and dies at the first invoke while every `-jit` gate stays green. **Weakening `check_exe_fn_global_restore` (`REVIEW.das`) - the gate that reads those initializers - is a defect.** -**A value that a team-lane body reads is a `def` returning it, or a module `let` of a bool, -number, enum or bitfield type whose initializer is a constant - never a module `var` or any other -module `let`.** A pooled lane runs in another context, so a module global it reads comes back zero; -the optimizer, when on, folds such a `let` into its literal at each read. +**A module-level value that a team-lane body reads is a `def` returning it, or a module `let` of a +bool, number, enum or bitfield type whose initializer is a constant - never a module `var` or any +other module `let`.** A pooled lane runs in another context, so a module global it reads comes back +zero; the optimizer, when on, folds such a `let` into its literal at each read. **Nothing in a team-lane body names a module global it writes or resizes - write into a buffer the dispatching caller sizes and passes in instead.** A pooled lane runs in another context, so a resize acts on that context's out-of-date copy of the global. **An array a `dasllama/` function resizes through the builtin `resize` or any helper that reaches -it (`reserve_resize`, `grow_resize`, `ensure_length`, `overwrite_resize` among them), whose element -count grows -with a count the model file sets and is not declared `@exact_size` (`@scratch @exact_size` when -the buffer is `@scratch`), is a defect.** -PERF032 - the lint that flags a `resize` with no reserve before it - checks only `@exact_size` -arrays. +it, whose element count grows with a count the model file sets and is not declared `@exact_size` +(`@scratch @exact_size` when the buffer is `@scratch`), is a defect.** PERF032 - the lint that flags +a `resize` with no reserve before it - checks only `@exact_size` arrays. **Every `resize` of a buffer in `dasllama/` whose element count grows with a count the model file sets follows a `reserve(n)` or `ensure_capacity(n)` whose `n` is the resized count, or goes @@ -66,9 +63,9 @@ its own. A plane whose byte count scales with the model and that the load reads gguf without a `FillJob` is the same defect. **A diff that adds a lasting allocation the path never made before, or adds a term to one's size, -that grows with a scaling count states that size in bytes in a `PERF_LEDGER.md` row: at the largest shape the code path -accepts, or, where the path accepts any value of the count, as a formula in the count with its -value at two shapes that differ in it.** +that grows with a scaling count states that size in bytes in a `PERF_LEDGER.md` row: at the largest +shape the code path accepts, or, where the path accepts any value of the count, as a formula in the +count with its value at two shapes that differ in it.** **A diff that changes the size formula of a lasting allocation that exists before and after it without adding a term - a count that starts or stops scaling it, a bound that becomes piecewise - diff --git a/modules/dasLLAMA/REVIEW_PLACEMENT.md b/modules/dasLLAMA/REVIEW_PLACEMENT.md index b1a9377b33..4bdc95f845 100644 --- a/modules/dasLLAMA/REVIEW_PLACEMENT.md +++ b/modules/dasLLAMA/REVIEW_PLACEMENT.md @@ -6,9 +6,8 @@ docs: `ARCHITECTURE.md`, `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_ENGINE_SERVING. `ARCHITECTURE_MEDIA.md`, `ARCHITECTURE_TTS.md`, `ARCHITECTURE_POCKET.md`. Planned work: `followup_general.md`, `followup_vulkan.md` for Vulkan, `followup_metal.md` for Metal. -A charter line is a file's line in the companion `File charters` section that -`ARCHITECTURE.md#file-charters` routes that file to, or its role row or file bullet in -`ARCHITECTURE_GPU.md#gpu-backends`. +A charter line is a file's line in the companion section that `ARCHITECTURE.md#file-charters` routes +that file to, or its role row or file bullet in `ARCHITECTURE_GPU.md#gpu-backends`. **A per-file inventory restated in this checklist is a defect of the checklist.** The charter lines own the per-file list; a rule naming what KIND of code lands in which file is the @@ -30,8 +29,9 @@ change that charter row in the same change.** row names; a concern the row names without a backend lands in any file of the row.** **A diff adding a new item of the sort a charter line's parenthetical lists (a model family, a -weight format, a dispatch arm) lands it in that charter line's file, whether or not the -parenthetical names it, and adds it to the parenthetical in the same change.** +weight format, a dispatch arm - and a function of the sort a parenthetical of function names lists) +lands it in that charter line's file, whether or not the parenthetical names it, and adds it to the +parenthetical in the same change.** **Never add a charter line for a file outside `dasllama/` - state what it holds in its own folder's architecture doc or `CLAUDE.md` instead.** @@ -87,9 +87,10 @@ home's kernels splice lands once - never as a copy per home - in **A method of a class template more than one home stamps lands with its template.** -**A disk-order -> compute-order transform lands by its consumer: a transform into the layout -the CPU kernels read row by row in `dasllama/dasllama_repack.das`, a transform into the layout a -GPU plane or gather reads in `dasllama/dasllama_layout.das`.** +**A disk-order -> compute-order transform lands by its consumer: a transform into the layout the CPU +kernels read row by row in `dasllama/dasllama_repack.das`, a transform into the layout a GPU plane +or gather reads in `dasllama/dasllama_layout.das`; a panel a batch call builds into scratch and +consumes in the same call is no such transform and lands beside the walk that builds it.** **A CPU KV-cache store, read, score dot, or V-accumulate OVER CACHE BYTES - a codec primitive that knows the K/V element format - lands in `dasllama/dasllama_kv_codec.das`.** A dot over an @@ -113,12 +114,12 @@ stamp it, whether or not the CUDA home does `check_shared_template_homes` is a defect. **A CPU kernel body under `dasllama/` - a function whose loop turns a weight format's packed bytes -into a dot product or a dequantized f32 row, a `[tune]` variant, a function a -`register_kernel_backend` call or a `KernelBackend` field default names, or a scalar body those -call - lands in a tier file, `dasllama/dasllama_math_.das`, never in -`dasllama/dasllama_math.das`; a body that a -`register_kernel_backend` call in another tier file names, or that two tier files' calls name, -lands in `dasllama/dasllama_math_default.das`.** +(block-quantized bytes with their scales, not an f16 or bf16 element plane) into a dot product or a +dequantized f32 row, a `[tune]` variant, a function a `register_kernel_backend` call or a +`KernelBackend` field default names, or a scalar body those call - lands in a tier file, +`dasllama/dasllama_math_.das`, never in `dasllama/dasllama_math.das`; a body that a +`register_kernel_backend` call in another tier file names, or that two tier files' calls name, lands +in `dasllama/dasllama_math_default.das`.** **A branch - one arm of an `if` or a `match` - that only one model family takes, in a file whose charter line names no concern of that family, lands in that family's own file @@ -134,8 +135,8 @@ calls that seat; a seat's first engine-side call lands by its concern's charter in the kernel's file, and that file's charter line names the condition that selects the branch.** -**Code two tower families both need that names no family type - compute, stage/read, or load -orchestration - lands in `dasllama/dasllama_tower.das`.** +**Code two tower families' files call directly that names no family type - compute, stage/read, or +load orchestration - lands in `dasllama/dasllama_tower.das`.** **No signature in `dasllama/dasllama_tower.das` takes a type that `dasllama/dasllama_audio.das`, `dasllama/dasllama_vision.das`, or a family file declares - the @@ -168,8 +169,9 @@ whose module always registers (`dasCUDA`), writes a plain require.** require in an engine file - the program root requires the registration module directly.** **A function-typed global that a job (a forked context) invokes, or that a serialized exe must -re-establish and no other file's `[init]` arms, lands in a `dasllama/` file beside the `[init]` -that establishes it.** The `[init]` is the only code that runs where the global arrives unset. +re-establish and no other file's `[init]` arms, lands in a `dasllama/` file beside the `[init]` that +establishes it; a global declared beside its public setter, filled from the registering file's +`[init]`, complies.** The `[init]` is the only code that runs where the global arrives unset. **A `dasllama/` module whose `[init]` registers a hook the engine dispatches through gets its side-effect require in the same change that adds it** - a registration no engine file reaches diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 91963bc76c..c0acd11511 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -2299,15 +2299,15 @@ (`dasllama_math_gen.das`) multiplies a native bf16 plane as the reference does - the activations rounded to bf16, the TMUL bf16 tile over a k-pair panel - and `matmul_bf16_batch` reaches it where a sidecar names an `amx_bf16` perm; the gemma-E per-layer projection and the bf16 vision patch planes ride it (the 2026-10-25 `PERF_LEDGER.md` - Granite Rapids entry: E4B pp512 359.8 -> 368.8 tok/s over the widen walk, the tile 3921 against the f32 plane's - 782 GFLOP/s at that shape). What is left: the Qwen3-ASR mmproj's conv weights are F16 (`a.conv2d.*.weight`, - `a.conv_out.weight`), its blocks run the q8 lane, so the conv front's 1123 ms of f32 tile on the Xeon reaches the - matrix unit only through an f16 operand - AMX-FP16 (Granite Rapids on, Sapphire Rapids off) or an f16-to-bf16 - panel (three mantissa bits under the reference's f16 x f16); and the family has no vector leg, so Zen4 - (`vdpbf16ps`, avx512bf16) and Graviton (`bfdot` / `bfmmla`, the arm bf16 extension) run the widen walk. Done = the - conv front's operand ruled and served through the family, `vdpbf16ps` and `bfdot` perms in `dasllama_gemm_gen.das` - over the same panel with the probe gating them, and the Qwen3-ASR and image-chat CPU rows re-minted on every board - box at the release re-mint. + Granite Rapids entry, `debug-jit`: E4B pp512 359.8 -> 368.8 tok/s over the widen walk against the committed + `records/gnr.json` E4B CPU pp512 row's 325.7, the tile 3921 against the f32 plane's 782 GFLOP/s at that shape). What + is left: the Qwen3-ASR mmproj's conv weights are F16 (`a.conv2d.*.weight`, `a.conv_out.weight`), its blocks run the + q8 lane, so the conv front's 1123 ms of f32 tile on the Xeon reaches the matrix unit only through an f16 operand - + AMX-FP16 (Granite Rapids on, Sapphire Rapids off) or an f16-to-bf16 panel (three mantissa bits under the reference's + f16 x f16); and the family has no vector leg, so Zen4 (`vdpbf16ps`, avx512bf16) and Graviton (`bfdot` / `bfmmla`, + the arm bf16 extension) run the widen walk. Done = the conv front's operand ruled and served through the family, + `vdpbf16ps` and `bfdot` perms in `dasllama_gemm_gen.das` over the same panel with the probe gating them, and the + Qwen3-ASR, gemma-4-E4B and image-chat CPU rows re-minted on every board box at the release re-mint. 241. **The bench exe's ASR and TTS cells print a clip's best of their reps alone.** `lcpp_bench --asr` and `--tts` keep the minimum over `-r` reps (`r.enc_ms = min(...)`, the sentence sums) and the record row carries that one wall, @@ -2316,14 +2316,14 @@ print every rep's wall beside the best (`rep i: ` lines) and the record row's `tests` entry carries `ms_reps`, the recorder and `test_bench_records_schema.das` reading it back, so a ledger entry can quote the min and max. -242. **The ASR decoder's attention wall on CPU is its K/V bytes, and the session's codec is f16.** On the Granite - Rapids board the Qwen3-ASR decode over the ~3000 audio tokens spends `attn_chain` 8.8 s of 11.1 s (451 steps) - under the f16 K/V codec the ASR session defaults to (`set_asr_kv`), and 4.1 s of 6.3 s under q8_0 (443 steps - the - transcript moves); in `harness/cpu_asr_race.das`'s K pass the box streams 30 GB/s over (kv head, slice) units and - 62 GB/s by slices over every kv head, and the chain re-ordered to the latter did not move (the 2026-10-25 - `PERF_LEDGER.md` Granite Rapids entry). The reference decodes its f16 K/V at about 32 ms a step (its 21610 ms gb1 - total less its 7304 ms encode over the 451 steps, the 2026-10-24 tile entry's `llama-mtmd-cli` figures) against - our 24.6 (11115 ms over 451). Done = a ruling on the ASR session's CPU codec - q8_0 by default with the parity - cells' bars re-cut on its transcripts, or f16 kept and the row closed - written into +242. **The ASR decoder's attention wall on CPU is its K/V bytes, and the session's codec is f16.** On the Granite Rapids + board the Qwen3-ASR decode over the ~3000 audio tokens spends `attn_chain` 8.8 s of 11.1 s (451 steps) under the f16 + K/V codec the ASR session defaults to (`set_asr_kv`), and 4.1 s of 6.3 s under q8_0 (443 steps - the transcript + moves); in `harness/cpu_asr_race.das`'s K pass the box streams 30 GB/s over (kv head, slice) units and 62 GB/s by + slices over every kv head, and the chain re-ordered to the latter did not move (the 2026-10-25 `PERF_LEDGER.md` + Granite Rapids entry, every figure `debug-jit`). The reference decodes its f16 K/V at about 32 ms a step (its 21610 + ms gb1 total less its 7304 ms encode over the 451 steps, the 2026-10-24 tile entry's `llama-mtmd-cli` figures) + against our 24.6 (11115 ms over 451). Done = a ruling on the ASR session's CPU codec - q8_0 by default with the + parity cells' bars re-cut on its transcripts, or f16 kept and the row closed - written into `ARCHITECTURE_MEASUREMENT_AUDIO.md` and the facade's default, and the Qwen3-ASR CPU rows re-minted on every board - box at the release re-mint. + box at the release re-mint. \ No newline at end of file diff --git a/modules/dasLLAMA/harness/_race_fixture.das b/modules/dasLLAMA/harness/_race_fixture.das index ebb68a23ed..507a794bed 100644 --- a/modules/dasLLAMA/harness/_race_fixture.das +++ b/modules/dasLLAMA/harness/_race_fixture.das @@ -23,6 +23,7 @@ struct RaceTally { //! `count` operands off an LCG seeded by `salt`: (u - 0.5) x amp + bias, u on [0, 1) in 2^-16 steps; `stir` joins the step's increment def race_fill(count, salt : int; amp, bias : float; stir : uint = 0u) : array { var xa : array + xa |> reserve(count) xa |> resize(count) var st = uint(salt) * 2654435761u + 12345u for (i in range(count)) { diff --git a/modules/dasLLAMA/harness/cpu_asr_race.das b/modules/dasLLAMA/harness/cpu_asr_race.das index d52d67f342..ff1707d758 100644 --- a/modules/dasLLAMA/harness/cpu_asr_race.das +++ b/modules/dasLLAMA/harness/cpu_asr_race.das @@ -1,17 +1,22 @@ options gen2 options _dasllama_internal = true -// The CPU ASR line's four in-process races, one line a candidate, best of --reps interleaved rounds: -// --arm bf16 the bf16 weight plane's routes against the f32 plane through matmul_batch at the gemma-E -// projection's pp512 shape, the Qwen3-ASR conv's and a vision patch class: the stamped -// bf16w family where the sidecar names a perm, the widen walk, the row nest it replaced -// --arm gelu gelu_erf on the float lattice against the double erfc body it replaced, in place over a -// tower's FFN plane -// --arm mel log_mel_qwen3a over a 60 s synthetic clip, one lane against the lanes, the two images -// compared element for element -// --arm kpass the decode attention's K pass over a model-sized f16 cache (28 layers x 3000 keys x -// 8 kv heads x 128, two q heads a kv head): the (kv head, slice) unit order against one -// slice over every kv head, one lane and all lanes +// The CPU ASR line's races in one process: the arms timed in turn, one warmup round each arm drops, then --reps +// rounds; a report row is one arm's best round, with its compare and the tune manifest the run compiled under. +// --arm bf16 baseline: the f32 plane through matmul_batch (the tile walk) over the bf16 weights widened; +// compared arms: the bf16 plane through matmul_bf16_batch on the stamped bf16w family (where the +// sidecar names a perm), on the widen walk and on the row nest it replaced; the compare region is +// SAMPLE outputs of the ntok x d image, the baseline held to a double dot over them, the widen and +// nest arms to the f32 bound, the family (its activations rounded to bf16) to the bf16 bound +// --arm gelu baseline: the double erfc body gelu_erf replaced, held to the erf series; compared arm: the +// float kernel; the compare region is SAMPLE elements of the plane +// --arm kpass baseline: the decode attention's K pass over (kv head, slice) units on one lane (28 layers x 3000 +// keys x 8 kv heads x 128, f16, two q heads a kv head), held to a double dot; compared arms: the +// same units on the lanes, one slice over every kv head on one lane and on the lanes, each +// bit-exact with the baseline (dot_f16 a score, the same call on every route); the region is +// SAMPLE scores +// ATTRIBUTION SWEEP --arm mel: log_mel_qwen3a over a 60 s synthetic clip, one lane against the lanes, +// timing-only - the lanes' image against the one-lane image is tests/test_whisper.das's mel cell // // DAS_TUNE_MANIFEST= bin/daslang -jit modules/dasLLAMA/harness/cpu_asr_race.das -- [--arm all] [--reps N] @@ -23,6 +28,8 @@ require dasllama/dasllama_qwen3a require dasllama/dasllama_audio require dasllama/dasllama_par require _race_fixture +require ../tests/_compares.das +require daslib/env_registry require daslib/jobque_boost require daslib/f16_cvt require daslib/clargs @@ -32,7 +39,7 @@ require math struct Args { @clarg_doc = "Which race: bf16 | gelu | mel | kpass | all (default: all)" arm : string = "all" - @clarg_doc = "Timed rounds a candidate, interleaved, the best kept (default: 5)" + @clarg_doc = "Timed rounds a candidate after its warmup round, interleaved, the best kept (default: 5)" reps : int = 5 @clarg_short = "?" @clarg_name = "show-help" @@ -40,111 +47,201 @@ struct Args { help : bool } -//! one candidate's line: its best round in ms, the median and the cv beside it, `rate` the figure the best makes -def private line(race, who : string; us : array; rate : block<(ms : float) : string>) { +let SAMPLE = 512 +let F32_BOUND = 1.0e-5 +let BF16_BOUND = 1.0e-2 +let GELU_BOUND = 1.0e-5 +let REF_BOUND = 1.0e-6 +let WARMUP = 1 + +//! the compare region: SAMPLE indices into a `total`-element image off an LCG +def private sample_idx(total : int) : array { + var idx : array + idx |> resize(SAMPLE) + var st = 0x9E3779B9u + for (i in range(SAMPLE)) { + st = st * 1664525u + 1013904223u + idx[i] = int((st >> 4u) % uint(total)) + } + return <- idx +} + +def private gather(src : array; idx : array) : array => <- [for (i in idx); src[i]] + +def private manifest_of() : string { + let m = env_value_of("DAS_TUNE_MANIFEST") + return empty(m) ? "manifest=unset" : "manifest={m}" +} + +//! one candidate's report row: the best round in ms, the median and cv beside it, `rate` the figure the best +//! makes, the compare's words; a miss prints the miss and the bound alone +def private row(race, who : string; us : array; h : Held; against : string; rate : block<(ms : float) : string>; var t : RaceTally) { + let st = stats(us) + let words = compare_words(h, against, "{SAMPLE} sampled elements") + if (!h.ok) { + print("{race} {who}: {words} {manifest_of()}\n") + tally(t, -1) + return + } + print("{race} {who}: {st.best} ms ({invoke(rate, st.best)}; median {st.median} ms, cv {st.cv}) {words} {manifest_of()}\n") + tally(t, 1) +} + +//! a timing-only row of an attribution sweep +def private row_timing_only(race, who : string; us : array; rate : block<(ms : float) : string>; var t : RaceTally) { let st = stats(us) - print("{race} {who}: {st.best} ms ({invoke(rate, st.best)}; median {st.median} ms, cv {st.cv})\n") + print("{race} {who}: {st.best} ms ({invoke(rate, st.best)}; median {st.median} ms, cv {st.cv}) timing-only {manifest_of()}\n") + tally(t, 1) } //! past every dot length: the row nest serves every output let private NEST_ONLY_MIN_N = 1l << 30 -def private race_bf16(n, d, ntok : int64; reps : int) { +//! every arm once - `before(i)`, `arm(i)`, then `sample(i)` reading its output - and then the timed rounds +def private sampled_rounds(narms, reps : int; before : block<(i : int) : void>; arm : block<(i : int) : void>; sample : block<(i : int) : void>) : array> { + for (i in range(narms)) { + invoke(before, i) + invoke(arm, i) + invoke(sample, i) + } + return <- time_rounds(narms, reps, WARMUP, before, arm) +} + +def private widen(h : uint16) : float => unsafe(reinterpret(uint(h) << 16u)) + +def private race_bf16(n, d, ntok : int64; reps : int; var t : RaceTally) { let w <- race_fill(int(n * d), 1, 1.2, 0.0) var w16 : array - w16 |> resize(n * d) - for (h, v in w16, w) { + w16 |> reserve_resize(n * d) + var wwide : array + wwide |> reserve_resize(n * d) + for (h, wide, v in w16, wwide, w) { h = bf16_bits(v) + wide = widen(h) } let x <- race_fill(int(ntok * n), 2, 1.4, 0.0) var y : array - y |> resize(ntok * d) + y |> reserve_resize(ntok * d) + let idx <- sample_idx(int(ntok * d)) + var want : array + want |> resize(SAMPLE) + for (s, wv in idx, want) { + let tk = int64(s) / d + let j = int64(s) % d + var acc = 0.0lf + for (k in range64(n)) { + acc += double(x[tk * n + k]) * double(wwide[j * n + k]) + } + wv = float(acc) + } let family = get_bf16_batch_gen() let min_n = get_f32_batch_tile_min_n() - let labels <- [family != null ? "bf16 plane, the stamped family" : "bf16 plane, the widen walk (no family stamped)", - "f32 plane, the tile walk", "bf16 plane, the widen walk", "bf16 plane, the row nest"] - var times <- time_rounds(length(labels), reps, 1, $(i : int) { - if (i == 0) { + var labels <- ["f32 plane, the tile walk"] + var arms : array // 0 the baseline, 1 the family, 2 the widen walk, 3 the row nest + arms |> push(0) + if (bf16w_batch_serves(n, d, ntok)) { + labels |> push("bf16 plane, the stamped family") + arms |> push(1) + } + labels |> push("bf16 plane, the widen walk") + arms |> push(2) + labels |> push("bf16 plane, the row nest") + arms |> push(3) + let narms = length(arms) + var got : array> + got |> resize(narms) + let times <- sampled_rounds(narms, reps, $(i : int) { + let a = arms[i] + if (a == 1) { set_bf16_batch_gen(family) } else { let none : Bf16BatchGenFn set_bf16_batch_gen(none) } - set_f32_batch_tile_min_n(i == 3 ? NEST_ONLY_MIN_N : min_n) + set_f32_batch_tile_min_n(a == 3 ? NEST_ONLY_MIN_N : min_n) + poison(y) }, $(i : int) { - if (i == 1) { - matmul_batch(y, w, x, n, d, ntok) + if (arms[i] == 0) { + matmul_batch(y, wwide, x, n, d, ntok) } else { matmul_bf16_batch(y, w16, 0l, x, n, d, ntok) } + }, $(i : int) { + got[i] <- gather(y, idx) }) set_bf16_batch_gen(family) set_f32_batch_tile_min_n(min_n) let fl = 2.0lf * double(n * d * ntok) / 1.0e6lf - for (lbl, us in labels, times) { - line("bf16 n={n} d={d} ntok={ntok}", lbl, us) $(ms : float) => "{fl / double(ms)} GFLOP/s" + for (i in range(narms)) { + let h = held(got[i], want, arms[i] == 1 ? BF16_BOUND : F32_BOUND) + row("bf16 n={n} d={d} ntok={ntok}", labels[i], times[i], h, "the double dot", $(ms : float) => "{fl / double(ms)} GFLOP/s", t) } } //! the GELU body before the float kernel: erfc as the NR rational fit over libm's exp, every op in double def private gelu_erf_double(var x : array) { for (v in x) { - let d = double(v) - let z = abs(d) * 0.7071067811865476lf - let t = 1.0lf / (1.0lf + 0.5lf * z) - let erfc = t * exp(-z * z - 1.26551223lf + t * (1.00002368lf + t * (0.37409196lf + t * (0.09678418lf + t * (-0.18628806lf - + t * (0.27886807lf + t * (-1.13520398lf + t * (1.48851587lf + t * (-0.82215223lf + t * 0.17087277lf))))))))) - v = float(d >= 0.0lf ? 0.5lf * d * (2.0lf - erfc) : 0.5lf * d * erfc) + let dv = double(v) + let z = abs(dv) * 0.7071067811865476lf + let tt = 1.0lf / (1.0lf + 0.5lf * z) + let erfc = tt * exp(-z * z - 1.26551223lf + tt * (1.00002368lf + tt * (0.37409196lf + tt * (0.09678418lf + tt * (-0.18628806lf + + tt * (0.27886807lf + tt * (-1.13520398lf + tt * (1.48851587lf + tt * (-0.82215223lf + tt * 0.17087277lf))))))))) + v = float(dv >= 0.0lf ? 0.5lf * dv * (2.0lf - erfc) : 0.5lf * dv * erfc) } } -def private race_gelu(reps : int) { +def private race_gelu(reps : int; var t : RaceTally) { let n = 1024 * 4096 let x0 <- race_fill(n, 3, 8.0, 0.0) var x : array - x |> resize(n) - let labels <- ["the float kernel", "the double body"] - var times <- time_rounds(length(labels), reps, 1, $(_i : int) { - for (d, s in x, x0) { - d = s + x |> reserve_resize(n) + let idx <- sample_idx(n) + var want : array + want |> resize(SAMPLE) + for (s, wv in idx, want) { + let dv = double(x0[s]) + wv = float(0.5lf * dv * (1.0lf + erf_series(dv * 0.7071067811865476lf))) + } + let labels <- ["the double body", "the float kernel"] + var got : array> + got |> resize(2) + let times <- sampled_rounds(2, reps, $(_i : int) { + for (dst, src in x, x0) { + dst = src } }, $(i : int) { if (i == 0) { - gelu_erf(unsafe(addr(x[0])), int64(n)) - } else { gelu_erf_double(x) + } else { + gelu_erf(unsafe(addr(x[0])), int64(n)) } + }, $(i : int) { + got[i] <- gather(x, idx) }) - for (lbl, us in labels, times) { - line("gelu_erf {n} elements", lbl, us) $(ms : float) => "{ms * 1.0e6 / float(n)} ns an element" + for (i in range(2)) { + let h = held(got[i], want, i == 0 ? REF_BOUND : GELU_BOUND) + row("gelu_erf {n} elements", labels[i], times[i], h, "the erf series", $(ms : float) => "{ms * 1.0e6 / float(n)} ns an element", t) } } -def private race_mel(reps : int) { +def private race_mel(reps : int; var t : RaceTally) { var samples : array samples |> reserve_resize(16000 * 60) for (i in range(16000 * 60)) { samples[i] = 0.3 * sin(float(i) * 0.01) + 0.1 * sin(float(i) * 0.37) + 0.05 * float((i * 7919) % 101) / 101.0 } - set_single_thread_(true) - var inscope a <- log_mel_qwen3a(samples, 128l) - set_single_thread_(false) - var inscope b <- log_mel_qwen3a(samples, 128l) + if (get_dispatch_lanes() <= 1) { + print("mel: WARNING one dispatch lane on this box - the single-thread switch changes nothing\n") + } let labels <- ["one lane", "the lanes"] - var times <- time_rounds(length(labels), reps, 1, $(i : int) { + let times <- time_rounds(2, reps, WARMUP, $(i : int) { set_single_thread_(i == 0) }, $(_i : int) { var inscope m <- log_mel_qwen3a(samples, 128l) }) set_single_thread_(false) - var diff = 0 - for (p, q in a.data, b.data) { - if (p != q) { - diff++ - } - } - for (lbl, us in labels, times) { - line("mel 60 s x 128", lbl, us) $(_ms : float) => "elements apart {diff} of {length(a.data)}" + for (i in range(2)) { + row_timing_only("mel 60 s x 128", labels[i], times[i], $(_ms : float) => "{get_dispatch_lanes()} lanes on the box", t) } } @@ -211,7 +308,7 @@ def private kpass_rows(kall : array; q : array; var att : array kall |> reserve_resize(int(KP_L * KP_KEYS * kvdim)) @@ -225,18 +322,48 @@ def private race_kpass(reps : int) { } var att : array att |> resize(2l * KP_KVH * KP_SEQ) - let mb = double(KP_L * KP_KEYS * kvdim * 2l) / 1.0e6lf + // the region: SAMPLE scores of the last layer's image, keys under KP_KEYS only + var idx : array + idx |> resize(SAMPLE) + var st = 0x9E3779B9u + for (i in range(SAMPLE)) { + st = st * 1664525u + 1013904223u + let hh = int((st >> 4u) % uint(2l * KP_KVH)) + st = st * 1664525u + 1013904223u + let j = int((st >> 4u) % uint(KP_KEYS)) + idx[i] = hh * int(KP_SEQ) + j + } + var want : array + want |> resize(SAMPLE) + let last = (KP_L - 1l) * KP_KEYS * kvdim + for (s, wv in idx, want) { + let hh = int64(s) / KP_SEQ + let j = int64(s) % KP_SEQ + var acc = 0.0lf + for (c in range64(KP_HS)) { + acc += double(q[hh * KP_HS + c]) * double(f16_to_f32(uint(kall[last + j * kvdim + (hh / 2l) * KP_HS + c]))) + } + wv = float(acc) + } let labels <- ["(kv head, slice) units, one lane", "(kv head, slice) units, {KP_KVH * KP_NSL} units on the lanes", "a slice over every kv head, one lane", "a slice over every kv head, {KP_NSL} units on the lanes"] - var times <- time_rounds(length(labels), reps, 1) $(i : int) { + var got : array> + got |> resize(4) + let times <- sampled_rounds(4, reps, $(_i : int) { + poison(att) + }, $(i : int) { if (i < 2) { kpass_heads(kall, q, att, i == 0 ? 1 : int(KP_KVH * KP_NSL)) } else { kpass_rows(kall, q, att, i == 2 ? 1 : int(KP_NSL)) } - } - for (lbl, us in labels, times) { - line("kpass {mb} MB of f16 keys a step", lbl, us) $(ms : float) => "{mb / double(ms)} GB/s, {ms / float(KP_L)} ms a layer" + }, $(i : int) { + got[i] <- gather(att, idx) + }) + let mb = double(KP_L * KP_KEYS * kvdim * 2l) / 1.0e6lf + for (i in range(4)) { + let h = i == 0 ? held(got[0], want, F32_BOUND) : held(got[i], got[0], 0.0) + row("kpass {mb} MB of f16 keys a step", labels[i], times[i], h, i == 0 ? "the double dot" : labels[0], $(ms : float) => "{mb / double(ms)} GB/s, {ms / float(KP_L)} ms a layer", t) } } @@ -253,21 +380,22 @@ def main() : int { print_help(get_command_info(type), "cpu_asr_race") return 0 } + var t : RaceTally with_dasllama_jobque_() { if (cfg.arm == "all" || cfg.arm == "bf16") { - race_bf16(2560l, 10752l, 512l, cfg.reps) - race_bf16(4320l, 480l, 800l, cfg.reps) - race_bf16(1152l, 1152l, 256l, cfg.reps) + race_bf16(2560l, 10752l, 512l, cfg.reps, t) + race_bf16(4320l, 480l, 800l, cfg.reps, t) + race_bf16(1152l, 1152l, 256l, cfg.reps, t) } if (cfg.arm == "all" || cfg.arm == "gelu") { - race_gelu(cfg.reps) + race_gelu(cfg.reps, t) } if (cfg.arm == "all" || cfg.arm == "mel") { - race_mel(cfg.reps) + race_mel(cfg.reps, t) } if (cfg.arm == "all" || cfg.arm == "kpass") { - race_kpass(cfg.reps) + race_kpass(cfg.reps, t) } } - return 0 + return race_exit(t, "cpu_asr_race") } diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 8cbe12e849..547ab64db3 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1032,6 +1032,36 @@ def bf16w_test_family(fxs : array) : bool { return allok } +typedef Bf16wCand = tuple + +//! the candidates' rounds in turn - one timed run of every candidate, then the next round, three rounds, the best +//! kept; no warmup round dropped, the gate run before already warmed each; empty where the gate failed +def private bf16w_time_candidates(cands : array; fx : Bf16wFixture; gated : bool; wvs : array>>; cvs : array>>; var y : array) : array { + var dts : array + dts |> resize(length(cands)) + for (dt in dts) { + dt = 1.0e30lf + } + if (!gated) return <- dts + for (_rep in range(3)) { + for (c, dt in cands, dts) { + if (c.form > 0l) { + arm_of(wvs, cvs, c.name) + } + let t0 = ref_time_ticks() + if (c.form > 0l) { + run_bf16w_variant(c.fn, fx, c.form, c.ts, y) + } else { + unsafe { // the reference row serves through the widen walk, not its own body + batch_tile_block(addr(y[0]), addr(fx.w16[0]), addr(fx.x[0]), fx.n, fx.d, 0l, fx.d, 0l, fx.ntok, fx.ntok) + } + } + dt = min(dt, double(get_time_usec(t0))) + } + } + return <- dts +} + //! the bf16-weight family's tune pass: every variant gated on the fixture, the tile-form variants and the widen walk //! (the `mr16` row's production route) timed on the gemma-E projection's pp512 shape, the fastest the entry def bf16w_tune_family(var rows_out : table) : string { @@ -1048,6 +1078,8 @@ def bf16w_tune_family(var rows_out : table) : string { var bestDt = 1.0e30lf var y : array y |> resize(fx.ntok * fx.d) + var cands : array + var allok = true for (v in tvs) { let form = int64(forms?[v._0] ?? 0) let ts = int64(steps?[v._0] ?? 4) @@ -1060,30 +1092,25 @@ def bf16w_tune_family(var rows_out : table) : string { var maxdiff = 0.0 run_bf16w_variant(v._1, fx, form, ts, y) if (!close_to_ref(fx, y, maxdiff, bf16w_bar_form(form))) { - print("bf16w {v._0}: MISMATCH (maxdiff {maxdiff}) - aborting tune\n") - best = "" + print("bf16w {v._0}: MISMATCH (maxdiff {maxdiff}, the bound {bf16_envelope(fx.yref, length(y), bf16w_bar_form(form))} past close_tol) - aborting tune\n") + allok = false break } - var dt = 1.0e30lf - for (_rep in range(3)) { - let t0 = ref_time_ticks() - if (form > 0l) { - run_bf16w_variant(v._1, fx, form, ts, y) - } else { - unsafe { // the reference row serves through the widen walk, not its own body - batch_tile_block(addr(y[0]), addr(fx.w16[0]), addr(fx.x[0]), fx.n, fx.d, 0l, fx.d, 0l, fx.ntok, fx.ntok) - } - } - dt = min(dt, double(get_time_usec(t0))) - } + tune_detail("bf16w {v._0} (form={form}, tokstep={ts}): ok (maxdiff {maxdiff}, within the bf16 envelope {bf16_envelope(fx.yref, length(y), bf16w_bar_form(form))})\n") + cands |> push((name = v._0, fn = v._1, form = form, ts = ts)) + } + var dts <- bf16w_time_candidates(cands, fx, allok, wvs, cvs, y) + names |> reserve(length(cands)) + times |> reserve(length(cands)) + for (c, dt in cands, dts) { let gf = 2.0lf * double(fx.n * fx.d * fx.ntok) / dt / 1000.0lf - rows_out[v._0] = JV((tile_us = dt)) - names |> push(v._0) + rows_out[c.name] = JV((tile_us = dt)) + names |> push(c.name) times |> push(dt) - tune_detail("bf16w {v._0} (form={form}, tokstep={ts}): ok (maxdiff {maxdiff}), {dt} us ({gf:0.1f} GFLOP/s)\n") + tune_detail("bf16w {c.name} (form={c.form}, tokstep={c.ts}): {dt} us ({gf:0.1f} GFLOP/s)\n") if (dt < bestDt) { bestDt = dt - best = v._0 + best = c.name } } if (!empty(best)) { @@ -1092,6 +1119,8 @@ def bf16w_tune_family(var rows_out : table) : string { delete names delete times delete y + delete cands + delete dts delete cvs delete wvs delete steps diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 8663957d0d..54eee1df30 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -7,8 +7,9 @@ options _dasllama_internal = true // [dasllama_grid] emits, cross-module, each template's __ variant clones plus a // _template_variants() fn-ptr registry; the bench runs the registry INTERLEAVED under the // JIT (loop hints only bite there — see dasllama_tune), correctness-gates every variant vs an f64 -// reference (exact equality for the requant), and reports each variant's time as a DELTA vs that -// kernel's SHIPPED fallback perm. +// reference over every element the kernel writes (exact equality for the requant; TOL relative +// elsewhere, a REJECTED row names it), and reports each variant's time as a DELTA vs that kernel's +// SHIPPED fallback perm - the baseline arm; a report row's rounds follow one warm round it drops. // // Run: bin/daslang -jit modules/dasLLAMA/harness/tune_kernels.das, then rebuild/rerun the model // with -jit to pick up the sidecar (it re-keys the JIT DLL cache). The debugging concession is @@ -420,7 +421,7 @@ def private print_race(kernel : string; names : array; med, best : array tune_detail("--- {kernel} ---\n") for (i in range(length(names))) { if (!ok[i]) { - tune_detail(" {names[i]}: REJECTED\n") + tune_detail(" {names[i]}: REJECTED - outside {TOL} relative of the f64 reference\n") continue } let delta = base > 0.0lf ? (med[i] - base) / base * 100.0lf : 0.0lf diff --git a/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json index 26943818cd..e08138d43e 100644 --- a/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json @@ -43,6 +43,7 @@ "iq3s4q8_tile_gen" : "mr8", "iq4nl32q8_tile_gen" : "mr8", "q8q8_tile_gen" : "mr8_budget", + "bf16w_tile_gen" : "mr16", "iq4nlq8_gemv_gen" : "mr8", "k4q8_tile_gen" : "mr8", "k5q8_tile_gen" : "mr8", diff --git a/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json index d8e0a91727..3953b4cf45 100644 --- a/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json @@ -43,6 +43,7 @@ "iq3s4q8_tile_gen" : "mr8", "iq4nl32q8_tile_gen" : "mr8", "q8q8_tile_gen" : "mr8_budget", + "bf16w_tile_gen" : "mr16", "iq4nlq8_gemv_gen" : "mr8", "k4q8_tile_gen" : "mr8", "k5q8_tile_gen" : "mr8", diff --git a/modules/dasLLAMA/performance/defaults/arm-sve.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-sve.tune-defaults.json index ad585f049d..fc0f464b20 100644 --- a/modules/dasLLAMA/performance/defaults/arm-sve.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/arm-sve.tune-defaults.json @@ -43,6 +43,7 @@ "iq3s4q8_tile_gen" : "mr8", "iq4nl32q8_tile_gen" : "mr8", "q8q8_tile_gen" : "mr8_budget", + "bf16w_tile_gen" : "mr16", "iq4nlq8_gemv_gen" : "mr8", "k4q8_tile_gen" : "mr8", "k5q8_tile_gen" : "mr8", diff --git a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json index 72a5dc3562..d11d3d375b 100644 --- a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json @@ -43,6 +43,7 @@ "iq3s4q8_tile_gen" : "dot_amx_bf16_width512_mr16_kstep1_nrsplit2", "iq4nl32q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "q8q8_tile_gen" : "dot_amx_bf16_width512_mr16_kstep1_nrsplit2", + "bf16w_tile_gen" : "dot_amx_bf16_width512_mr16_kstep1_nrsplit2", "iq4nlq8_gemv_gen" : "dot_amx_bf16_width512_mr16_kstep1_nrsplit2", "k4q8_tile_gen" : "dot_amx_bf16_width512_mr16_kstep1_nrsplit2", "k5q8_tile_gen" : "dot_amx_bf16_width512_mr16_kstep1_nrsplit2", diff --git a/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json index 3487546f78..d8b23fbfc0 100644 --- a/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json @@ -43,6 +43,7 @@ "iq3s4q8_tile_gen" : "dot_maddubs_width256_mr8", "iq4nl32q8_tile_gen" : "dot_maddubs_width256_mr8", "q8q8_tile_gen" : "dot_maddubs_width256_mr8_kstep2", + "bf16w_tile_gen" : "mr16", "iq4nlq8_gemv_gen" : "dot_maddubs_width256_mr8", "k4q8_tile_gen" : "dot_maddubs_width256_mr8", "k5q8_tile_gen" : "dot_maddubs_width256_mr8", diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json index 2a0910700e..aad4f86485 100644 --- a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -43,6 +43,7 @@ "iq3s4q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "iq4nl32q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "bf16w_tile_gen" : "mr16", "iq4nlq8_gemv_gen" : "dot_vpdpbusd_width512_mr16", "k4q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", From 822d20f7b924743a9fb620c6237bef7c662a8745 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sat, 24 Oct 2026 23:20:57 -0700 Subject: [PATCH 18/18] dasLLAMA: the bf16 arc's ledger entry with the four boards' figures The 2026-10-25 entry carries every board box's race rows (the compares within their bounds), the Qwen3-ASR and gemma-4-E4B CPU rows the rig exes minted into a scratch store beside the release-17 board figures, the image encode's open gap traced to master, and the byte formulas of the buffers the bf16 plane's calls now fill; rows 240 and 242 name the re-mint run and the entry each figure comes from; the charter bullets the rewrap had flattened are back. The audio, vision, kq-formats and upstream checklists carry their self-review fixes, and the architecture section states the bf16 mechanism in our terms. Co-Authored-By: Claude Fable 5.1 --- modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md | 9 +- .../dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md | 87 ++++---- modules/dasLLAMA/PERF_LEDGER.md | 200 ++++++++++-------- modules/dasLLAMA/REVIEW_AUDIO.md | 7 +- modules/dasLLAMA/REVIEW_KQ_FORMATS.md | 19 +- modules/dasLLAMA/REVIEW_VISION.md | 22 +- .../dasLLAMA/dasllama/dasllama_math_gen.das | 2 +- modules/dasLLAMA/dasllama/dasllama_qwen3a.das | 2 +- modules/dasLLAMA/followup_general.md | 33 +-- 9 files changed, 210 insertions(+), 171 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md index 42c56dbd73..8758ef9e41 100644 --- a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -211,12 +211,11 @@ another expert than decode would (`test_parity.das` holds the 26B-A4B's Q4_0 fil tokens on the CPU backend; `test_matmul_batch.das` holds the entry to the per-token matmul bit for bit at the router's shape). -### A bf16 plane multiplies through the matrix unit as the reference does {#bf16-weight-tile-family} +### A bf16 plane multiplies through the matrix unit on bf16 activations {#bf16-weight-tile-family} -The reference build multiplies a bf16 weight plane - an encoder tower's GEMMs, gemma-E's per-layer -projection - as bf16 times bf16: it rounds the f32 activations to bf16 first. `bf16w_batch` does the -same through the `bf16w_tile_gen` family where the family serves the call; a declined call -multiplies the f32 activations unrounded. Once per call, `xbf16_build_f32` rounds the activations to +A bf16 weight plane - an encoder tower's GEMMs, gemma-E's per-layer projection - multiplies as bf16 +times bf16, the f32 activations rounded to bf16 first: `bf16w_batch` runs the `bf16w_tile_gen` family +where the family serves the call; a declined call multiplies the f32 activations unrounded. Once per call, `xbf16_build_f32` rounds the activations to bf16 (nearest even) and also keeps the rounded values widened back to f32. For each unit, `bf16w_panel_build` packs the plane's rows into the B-tile layout the quant families' diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md index 7e8d3ce1bd..bd187c1009 100644 --- a/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md +++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE_FORMATS.md @@ -245,47 +245,48 @@ repack walks that plane's region alone (`shexp_plane_q8`). ### CPU kernel tiers {#cpu-kernel-tiers} - **`dasllama_math.das`** - the numeric ABSTRACTION: typedefs, active backend pointers, public -wrappers, dispatch shaping, and the array-sizing helpers every tier and the engine share -(`reserve_resize`, `grow_resize`, `ensure_length`, `overwrite_resize`), the `[tuned]` fp32 loop-hint -families (`dot`, `axpy`, the elementwise set, `f32_batch_tile`) and the fp32 batch walk -(`matmul_batch_core`, `batch_tile_block`, generic over the f32 and bf16 weight lattices - -`bf16_batch_tile` widens each halfword by the exact shift); every other kernel lives in a tier file. + wrappers, dispatch shaping, and the array-sizing helpers every tier and the engine share + (`reserve_resize`, `grow_resize`, `ensure_length`, `overwrite_resize`), the `[tuned]` fp32 + loop-hint families (`dot`, `axpy`, the elementwise set, `f32_batch_tile`) and the fp32 batch walk + (`matmul_batch_core`, `batch_tile_block`, generic over the f32 and bf16 weight lattices - `bf16_batch_tile` + widens each halfword by the exact shift); every other kernel lives in a tier file. - **`dasllama_math_default.das`** - the portable backend, always registered, always correct, -out-ranked by any platform tier - the per-32 rail's portable kernels and scalar leaves, which seat -every backend's unset per-32 slot (`#per-32-rail`) - and the ONE body of every Q8*Q8 and mx4 kernel -shape: each shape is a `def template` over a dot and a chunk-split placeholder, and -`[from_template]` stamps it per (dot, split) and per weight-scale plane (the f32 plane and the -wscale_f16 halfword plane, bit-identical over the widened plane). The idot4 stamps the arm64 tier -registers live here too: the `idot4` builtin is plain das that every target lowers for itself. - -**`dasllama_math_aarch64_neon.das`** - the arm64 lane-indexed SDOT tier: the `sdot4_laneq` / `tbl16` -composers only, plus the registration of the idot4 stamps as "arm64-sdot". Its `[init]` never fires -off-arch (`ARCHITECTURE_INVARIANTS.md#inherited-invariants`, three-layer safety model), so an -intrinsic here needs a correct scalar fallback body, not a guard at the call site. - -**`dasllama_math_accelerate.das`** - the Accelerate/BNNS float tier (AMX on M1-M3, SME on M4+), for -genuinely-float planes only. BLAS-for-quant is ruled out structurally -(`ARCHITECTURE_INVARIANTS.md#inherited-invariants`). - **`dasllama_math_gen.das`** / -**`dasllama_gemm_gen.das`** / **`dasllama_gemm_schema.das`** / **`dasllama_gemm_register.das`** - -the generated GEMM tier: the runtime registration, the tile generator, the layout/perm schema shared -by generator and runtime, and the `[tune]` family registration. A hand-written tile that the -generator could emit belongs in the generator. The bf16 tile's operand builders - the activation -widen (`xbf16_build`, its `bf16_bits` narrow, the f32 plane's `xbf16_build_f32`), the row-dequant -reference panel (`kq_panel_rows_bf16`) and the native bf16 plane's k-pair panel -(`bf16w_panel_build`) - sit in `dasllama_math_gen.das` beside the walk that runs them once per batch -call, in its JIT partition (`ARCHITECTURE_CPU_KERNELS.md#jit-partition-inlining`). The JIT cache -keys a generated kernel by its stub's hash and its perm's arguments, never by the generator's text, -so `REVIEW.das`'s `check_gemm_gen_stamp` pins one hash over `dasllama_gemm_gen.das` and -`dasllama_gemm_schema.das` with their comments and blank lines cut: a code edit to either is red -until the stamp moves, and the finding asks for the `LLVM_JIT_CODEGEN_VERSION` bump that re-keys the -cached kernels. The check licenses no names. - **`dasllama_tune.das`** - the per-box loop-hint tuner -(`[tuned]` / `[dasllama_grid]`) and the perm-less clone `[from_template]` (a template body into an -empty stub, placeholder calls renamed to the annotation's targets - the kernel shapes' stamp). -Tuning POLICY lives here; tuned VALUES live in the box's sidecar, never in source. The framework is -OPTIONAL: `llvm_tune` comes through the `tune_framework` group, empty in a build that did not -configure dasLLVM, and every use of it sits behind `static_if (typeinfo module_exists(llvm_tune))`, -so a build without dasLLVM opens no sidecar, reads no policy env and reports nothing - `[tuned]` -stamps the declared `fallback=` (or `DEFAULT_PERM`) and the AST rewrite is unchanged. -`box_profile_verdict_at` answers the reason NAME rather than llvm_tune's enum for the same reason: a -signature cannot be gated. - **`dasllama_tune_scope.das`** - the one `[tune_scope]` declaration -(tuner, covered modules, version pin, shipped defaults), in a module every kernel module requires: a -kernel reads its class entry in the defaults profile at its own compile, so the scope must be on the -AST first. \ No newline at end of file + out-ranked by any platform tier - the per-32 rail's portable kernels and scalar leaves, which + seat every backend's unset per-32 slot (`#per-32-rail`) - and the ONE body of every Q8*Q8 and + mx4 kernel shape: each shape is a `def template` over a dot and a chunk-split placeholder, and `[from_template]` stamps + it per (dot, split) and per weight-scale plane (the f32 plane and the wscale_f16 halfword plane, + bit-identical over the widened plane). The idot4 stamps the arm64 tier registers live here too: + the `idot4` builtin is plain das that every target lowers for itself. +- **`dasllama_math_aarch64_neon.das`** - the arm64 lane-indexed SDOT tier: the `sdot4_laneq` / + `tbl16` composers only, plus the registration of the idot4 stamps as "arm64-sdot". Its `[init]` + never fires off-arch (`ARCHITECTURE_INVARIANTS.md#inherited-invariants`, three-layer safety model), so an + intrinsic here needs a correct scalar fallback body, not a guard at the call site. +- **`dasllama_math_accelerate.das`** - the Accelerate/BNNS float tier (AMX on M1-M3, SME on M4+), + for genuinely-float planes only. BLAS-for-quant is ruled out structurally + (`ARCHITECTURE_INVARIANTS.md#inherited-invariants`). +- **`dasllama_math_gen.das`** / **`dasllama_gemm_gen.das`** / **`dasllama_gemm_schema.das`** / + **`dasllama_gemm_register.das`** - the generated GEMM tier: the runtime registration, the tile + generator, the layout/perm schema shared by generator and runtime, and the `[tune]` family + registration. A hand-written tile that the generator could emit belongs in the generator. + The bf16 tile's operand builders - the activation widen (`xbf16_build`, its `bf16_bits` + narrow, the f32 plane's `xbf16_build_f32`), the row-dequant reference panel + (`kq_panel_rows_bf16`) and the native bf16 plane's k-pair panel (`bf16w_panel_build`) - sit in + `dasllama_math_gen.das` beside the walk that runs them once per batch call, in its JIT + partition (`ARCHITECTURE_CPU_KERNELS.md#jit-partition-inlining`). The JIT cache keys a + generated kernel by its stub's hash and its perm's arguments, never by the generator's text, + so `REVIEW.das`'s `check_gemm_gen_stamp` pins one hash over `dasllama_gemm_gen.das` and + `dasllama_gemm_schema.das` with their comments and blank lines cut: a code edit to either is red until the stamp + moves, and the finding asks for the `LLVM_JIT_CODEGEN_VERSION` bump that re-keys the cached + kernels. The check licenses no names. +- **`dasllama_tune.das`** - the per-box loop-hint tuner (`[tuned]` / `[dasllama_grid]`) and the + perm-less clone `[from_template]` (a template body into an empty stub, placeholder calls + renamed to the annotation's targets - the kernel shapes' stamp). Tuning + POLICY lives here; tuned VALUES live in the box's sidecar, never in source. The framework is + OPTIONAL: `llvm_tune` comes through the `tune_framework` group, empty in a build that did not + configure dasLLVM, and every use of it sits behind `static_if (typeinfo module_exists(llvm_tune))`, so a build without dasLLVM opens no + sidecar, reads no policy env and reports nothing - `[tuned]` stamps the declared `fallback=` + (or `DEFAULT_PERM`) and the AST rewrite is unchanged. `box_profile_verdict_at` answers the + reason NAME rather than llvm_tune's enum for the same reason: a signature cannot be gated. +- **`dasllama_tune_scope.das`** - the one `[tune_scope]` declaration (tuner, covered modules, + version pin, shipped defaults), in a module every kernel module requires: a kernel reads its + class entry in the defaults profile at its own compile, so the scope must be on the AST first. diff --git a/modules/dasLLAMA/PERF_LEDGER.md b/modules/dasLLAMA/PERF_LEDGER.md index 5c8a51dc05..d1eaf846a6 100644 --- a/modules/dasLLAMA/PERF_LEDGER.md +++ b/modules/dasLLAMA/PERF_LEDGER.md @@ -10,92 +10,124 @@ and coverage; this ledger is the backlog for the perf pass that follows them. Ev what it costs today and what the fix would change. ## Entries -- **MEASURED (2026-10-25, `debug-jit`, `direction-grade`) - the float GELU, the bf16 plane walk, the AMX bf16 weight - tile, the mel's lanes and the decoder's K/V bytes, on the three AWS boards.** Every figure here is a `-jit` script - run (the three boards' rows excepted), `DAS_TUNE_POLICY` unset, `DAS_TUNE_MANIFEST` the box's rig release sidecar - under `performance/_rig/dasllama-bench/` (the M5: `performance/m5.tune.json`), `DASLLAMA_ALLOW_UNTUNED=1`, branch - `bbatkin/bf16-tower-gemm` at the commit named. The boards' Qwen3-ASR CPU rows at 3f7134f8f: each box's rig exe - re-released at that tip (`daspkg release --quick`, no policy override, the sidecar re-minted - the Granite Rapids - box's names `bf16w_tile_gen` = `dot_amx_bf16_width512_mr16_kstep1_nrsplit2`) and `performance/gen_bench_records.das - -- --workload asr -o Qwen3-ASR` run on it, the references `ARCHITECTURE_MEASUREMENT_AUDIO.md#asr-cpu-references`, - every cell the best of `--asr-reps` 2 (the exe prints a clip's best alone, `followup_general.md` row 241), das - against `llama-mtmd-cli` on gb1 (198.7 s), total / encode ms, every `was` the 2026-10-24 tile entry's ea6d942c2 row: - gnr (AWS m8i, Xeon 6975P-C, 8 cores) 13807 / 2007 against 20857 / 7481 (1.51x; was 15797 / 2917 against 21610 / - 7304), epyc9r14 (AWS m7a, 16 cores) 11694 / 2499 against 15916 / 8009 (1.36x; was 12784 / 3071 against 15981 / - 8051), graviton4 (AWS m8g, 16 cores) 8406 / 2462 against 67900 / 64309 (8.1x; was 9209 / 3168 against 67958 / - 64362); none published, the release re-mint owes the board (row 238). The Xeon's stage split - (`harness/asr_stage_probe.das -- -m Qwen3-ASR-0.6B-Q8_0.gguf --mmproj mmproj-Qwen3-ASR-0.6B-bf16.gguf -w gb1.wav -r - 1 --lang auto`, `JOBQUE_PROFILING=1 DAS_LOG_LEVEL=info -no-module-cache` for the engine buckets, gb1): at 7d7e39fc7 - (the GELU kernel and the bf16 walk) against ea6d942c2 (the 2026-10-24 tile entry's split) the encode 2451 against - 2924 ms, `enc.ffn.gelu` 41 against 194, `q3a.conv.mm` 1123 against 1446, `q3a.blocks` 588 against 726, `q3a.mel` 396 - against 403 (the mel's frame split is 258ca3799e; the race below carries it). `q3a.conv.mm` is GEMM-bound: 200 - chunks of a 3.3 GFLOP c2 and a 0.86 GFLOP c3 are 840 GFLOP at 750 GFLOP/s of f32 tile on 8 cores, against the race's - 1000 hot; the mmproj's conv weights are F16 and its blocks run the q8 lane, so the bf16 family reaches the conv - front only through an f16 ruling (row 240). `harness/cpu_asr_race.das -- --reps 5` (`-no-module-cache`, one process - a box, interleaved best of 5) at 24db4942af: the `bf16` arm, the bf16 plane through `matmul_bf16_batch` against the - f32 plane through `matmul_batch`, GFLOP/s at n d ntok 2560 10752 512 / 4320 480 800 / 1152 1152 256 - gnr on the - stamped AMX family 3921 / 3041 / 4093 against 782 / 923 / 1019; epyc9r14 on the widen walk 1667 / 1638 / 1537 - against 1610 / 1618 / 1618; graviton4 on the widen walk 664 / 599 / 657 against 599 / 563 / 636; the M5 Max on the - widen walk 978 / 846 / 672 against 988 / 786 / 761 (the dot_bf16 nest the walk replaces read 86 GFLOP/s on gnr at - the conv shape, the 2026-10-24 entry's nest arm). The `mel` arm (`log_mel_qwen3a` over a 60 s synthetic clip, 128 - mels; one lane / the lanes, ms): gnr 98.4 / 16.5, epyc9r14 101.0 / 15.1, graviton4 105.2 / 13.1, M5 52.2 / 10.2, 0 - of 768128 elements apart on every box. The `kpass` arm (the decode attention's K pass over 28 layers x 3000 keys x 8 - kv heads x 128, f16, two q heads a kv head, `dot_f16` a key a head; GB/s one lane / all lanes): (kv head, slice) - units gnr 3.6 / 30.3, epyc9r14 2.9 / 35.2, graviton4 8.4 / 87.5, M5 15.6 / 139.4; one slice over every kv head gnr - 9.1 / 62.1, epyc9r14 13.4 / 69.0, graviton4 9.6 / 102.0, M5 23.1 / 200.0. gemma-4-E4B Q8_0 pp512 on gnr - (`benchmarks/lcpp_bench.das -- -m gemma-4-E4B-it-Q8_0.gguf -p 512 -n 0 -r 3 -t 8 --for-debug-purposes`, the file - `unsloth/gemma-4-E4B-it-GGUF` at `resolve/main`): master 2d7d99fbf 284.7 +/- 18.3 tok/s (cv 6.4%, void), - 24e19e9916's kernels on the widen walk 359.8 +/- 0.3, the same with the AMX entry in the sidecar 368.8 +/- 0.2, the - frozen fixture's 8 opening tokens matching on all three. The decode on gnr at 24e19e9916's kernels (the stage probe, - the engine buckets on): f16 K/V (the ASR session's default) 11115 ms over 451 steps, `attn_chain` 8843 of it, - `blk_ffn` 1843, `mm_gemm` 525; `--kv q8` (the q8_0 K/V codec) 6285 ms over 443 steps, `attn_chain` 4097 - the - transcript moves (row 242). The chain's units re-ordered to one slice over every kv head (fcfb1dacd8) read - `attn_chain` 8942 ms against 24e19e9916's 8724 on gnr - within the run's noise; the kpass arm's two-fold does not - reach the chain, whose wall sits elsewhere. +- **MEASURED (2026-10-25, `debug-jit`, `direction-grade`) - the float GELU, the bf16 plane on the tile walk, the AMX + bf16 weight tile, the mel's lanes and the decoder's K/V bytes, on the four board boxes.** The author's board boxes: m5 + (M5 Max, 18 lanes), gnr (AWS m8i, Xeon 6975P-C, 8 cores), epyc9r14 (AWS m7a, Zen 4, 16 cores), graviton4 (AWS m8g, + Neoverse V2, 16 cores). Every figure but the board rows is a `-jit` script run on branch `bbatkin/bf16-tower-gemm` at + the commit named, `DAS_TUNE_POLICY` unset, `DAS_TUNE_MANIFEST` the box's rig release sidecar + (`performance/_rig/dasllama-bench*/` after `daspkg release --quick` at that commit), `-no-module-cache`; the board + rows are the rig exe's, `performance/gen_bench_records.das` with `--store ` (the three AWS boxes hold no + `llama-bench`, so their das rows pair with no reference), rows 240 and 242 own their re-mint; none enters + `performance/records/`. + + The Qwen3-ASR CPU rows (`-- --workload asr -o Qwen3-ASR`, the references + `ARCHITECTURE_MEASUREMENT_AUDIO.md#asr-cpu-references`, every cell the best of `--asr-reps` 2 - the exe prints a + clip's best alone, row 241 - das against `llama-mtmd-cli` on gb1 (198.7 s), total / encode ms; the AWS rows at + 3f7134f8f with every `was` the 2026-10-24 tile entry's ea6d942c2 row, the M5 row at cbfe4d670f with its `was` the + 2026-10-22 release-17 entry's d71b1aacc board row): gnr 13807 / 2007 against 20857 / 7481 (1.51x; was 15797 / 2917 + against 21610 / 7304), epyc9r14 11694 / 2499 against 15916 / 8009 (1.36x; was 12784 / 3071 against 15981 / 8051), + graviton4 8406 / 2462 against 67900 / 64309 (8.1x; was 9209 / 3168 against 67958 / 64362), m5 cpu tuned 8675 / 2475 + and cpu accel 7913 / 1608 against 46465 / 42396 (5.4x and 5.9x; was 13092 / 6460 and 10546 / 3856 against 46394 / + 42193). + + The gemma-4-E4B Q8_0 CPU rows at cbfe4d670f (`-- --workload llm -o gemma-4-E4B-it-Q8_0`, `-- --workload image -o + gemma-4-E4B-it-Q8_0`; the PLE projection and the BF16 mmproj ride the bf16 plane; `was` the 2026-10-22 release-17 + entry's board rows): gnr (3f7134f8f) pp512 375.3 +/- 24.7 tok/s (cv 6.6 %, void - the box's pp512 reads 6 % spreads on + every run), tg128 17.88 +/- 0.27, tg128@4 61.46 +/- 0.08 (was 325.7 / 18.1 / 61.7); m5 cpu tuned pp512 207.1 +/- 2.6, + tg128 53.11 +/- 0.08, tg128@4 126.6 +/- 2.3 (was 202.4 / 52.5 / 125.4), cpu accel 217.7 +/- 4.5 / 53.1 / 126.6 (was + 206.6 / 53.5 / 125.8); the image-chat cell on the m5 (coco 000000039769, 130 soft tokens, the tower on the q8 lane, + best of 3) cpu tuned encode 420 ms, pp 223, tg 51.5 tok/s (was 375 / 220.6 / 52.6), cpu accel 452 / 224.5 / 52.1 (was + 442.6 / 217.6 / 52.2): the encode's 420 against 375 is not this diff's - master 2d7d99fbff reads 427 ms and the tip + 422 under `benchmarks/lcpp_bench.das -- -m gemma-4-E4B-it-Q8_0.gguf --image-mmproj mmproj-gemma-4-E4B-it-BF16.gguf + --image coco-val2017-000000039769.jpg -r 3 -t 6 --for-debug-purposes` (the master run from a detached worktree at + origin/master, the same binary), an open gap against the release-17 board for the release re-mint (row 240). No other + board box of the author's can mint the image-chat cell (gnr lacks the E4B mmproj; epyc9r14 and graviton4 hold no E4B). + + The Xeon's stage split (`harness/asr_stage_probe.das -- -m Qwen3-ASR-0.6B-Q8_0.gguf --mmproj + mmproj-Qwen3-ASR-0.6B-bf16.gguf -w gb1.wav -r 1 --lang auto`, `JOBQUE_PROFILING=1 DAS_LOG_LEVEL=info` for the engine + buckets, gb1): at 7d7e39fc7 (the GELU kernel and the bf16 walk) against ea6d942c2 (the 2026-10-24 tile entry's split) + the encode 2451 against 2924 ms, `enc.ffn.gelu` 41 against 194, `q3a.conv.mm` 1123 against 1446, `q3a.blocks` 588 + against 726, `q3a.mel` 396 against 403 (the mel's frame split is 258ca3799e; the race below carries it). `q3a.conv.mm` + is GEMM-bound: 200 chunks of a 3.3 GFLOP c2 and a 0.86 GFLOP c3 are 840 GFLOP at 750 GFLOP/s of f32 tile on 8 cores, + against the race's 1000 hot; the mmproj's conv weights are F16 and its blocks run the q8 lane, so the bf16 family + reaches the conv front only through an f16 ruling (row 240). + + `harness/cpu_asr_race.das -- --arm all --reps 5` at cbfe4d670f (one process a box, the arms in turn, one warmup round + dropped, the best round; every compared arm within its bound on 512 sampled elements - the baselines to a double + reference, the family to the bf16 envelope, the K-pass orders bit-exact with each other; the PR body carries the + rows). The `bf16` arm, GFLOP/s at n d ntok 2560 10752 512 / 4320 480 800 / 1152 1152 256, the f32 plane on the tile + walk / the bf16 plane on the stamped family / the widen walk / the row nest it replaced: gnr 794 / 3740 / 959 / 91, + 1013 / 2083 / 980 / 90, 1042 / 2891 / 870 / 323; epyc9r14 1425 / - / 1551 / 442, 1413 / - / 1435 / 511, 1449 / - / + 1354 / 526; graviton4 598 / - / 662 / 192, 563 / - / 599 / 153, 660 / - / 664 / 295; m5 1474 / - / 1394 / 666, 1453 / + - / 1270 / 625, 1558 / - / 1340 / 658 (the family declines off AMX bf16). The `gelu` arm, ns an element over 4194304, + the double body it replaced against the float kernel: gnr 9.06 against 0.78, epyc9r14 5.41 against 0.79, graviton4 + 11.28 against 2.12, m5 3.34 against 0.83. The `mel` arm (`log_mel_qwen3a` over a 60 s synthetic clip, 128 mels, + timing-only; one lane / the lanes, ms): gnr 99.4 / 17.1 (8 lanes), epyc9r14 105.4 / 11.9 (16), graviton4 106.1 / 13.0 + (16), m5 56.0 / 6.9 (18); the lanes' image is the one-lane image element for element (`tests/test_whisper.das`, the + mel cell). The `kpass` arm (the decode attention's K pass over 28 layers x 3000 keys x 8 kv heads x 128, f16, two q + heads a kv head, `dot_f16` a score; GB/s one lane / all lanes): (kv head, slice) units gnr 3.4 / 26.6, epyc9r14 2.6 / + 30.8, graviton4 8.1 / 85.8, m5 9.8 / 123.1; one slice over every kv head gnr 9.2 / 54.4, epyc9r14 12.3 / 70.7, + graviton4 9.5 / 101.3, m5 20.3 / 186.4. + + gemma-4-E4B Q8_0 pp512 on gnr (`benchmarks/lcpp_bench.das -- -m gemma-4-E4B-it-Q8_0.gguf -p 512 -n 0 -r 3 -t 8 + --for-debug-purposes`, the file `unsloth/gemma-4-E4B-it-GGUF` at `resolve/main`): master 2d7d99fbf 284.7 +/- 18.3 + tok/s (cv 6.4 %, void), 24e19e9916's kernels on the widen walk 359.8 +/- 0.3, the same with the AMX entry in the + sidecar 368.8 +/- 0.2, the frozen fixture's 8 opening tokens matching on all three. The decode on gnr at 24e19e9916's + kernels (the stage probe, the engine buckets on): f16 K/V (the ASR session's default) 11115 ms over 451 steps, + `attn_chain` 8843 of it, `blk_ffn` 1843, `mm_gemm` 525; `--kv q8` (the q8_0 K/V codec) 6285 ms over 443 steps, + `attn_chain` 4097 - the transcript moves (row 242). The chain's units re-ordered to one slice over every kv head + (fcfb1dacd8) read `attn_chain` 8942 ms against 24e19e9916's 8724 on gnr - within the run's noise; the slice order's + two-fold over the (kv head, slice) units in the K pass does not reach the chain, whose wall sits elsewhere. + + The lasting buffers the bf16 plane's batch calls now fill, bytes as formulas in ntok and n with their values at the + race's 2560 x 512 and 4320 x 800 shapes: `g_bf16w_xr` (new) 4 ntok n - 5.24 MB, 13.8 MB; `g_kq_xbf16` 2 (ntok n + 32) + - 2.62 MB, 6.91 MB (the quant families' plane, now the bf16 plane's too); `g_walk_panel` where the family serves, + (round_up(32 form n, 64) + 64) slots + 64 at form 2 - 163968 and 276608 bytes a dispatch slot, 1.31 MB and 2.21 MB + over gnr's 8 slots. - **MEASURED (2026-10-24, `direction-grade`) - the fp32 batch GEMM tile and the lane-split conv front on the three AWS - boards.** `matmul_batch_core` runs a four-by-four tile (`f32_batch_tile`, - `ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice`); `harness/f32_batch_race.das -- --reps 5` races its two - constants and the tile against the dot-a-cell nest it replaces in one process, interleaved, best of 5 (`bin/daslang - -jit`, `DAS_TUNE_POLICY` unset and no other override; GFLOP/s; the M5 Max under `DAS_TUNE_MANIFEST` naming a scratch - sidecar the tuner minted for this kernel - `f32_batch_tile` vec4_u4 - with `DASLLAMA_ALLOW_UNTUNED=1` for the rest, - the Granite Rapids box - AWS m8i, Xeon 6975P-C, 8 cores - under its rig's minted sidecar, vec16_u2). The X block at - n 4320 d 480 ntok 800: M5 1463 / 1461 / 1431 / 1448 / 1441 / 1419 and Xeon 1067 / 1075 / 1053 / 1049 / 947 / 349 at - 64K / 128K / 256K / 512K / 1M / 4M; at n 1152 d 480 ntok 1536: M5 1431 / 1466 / 1497 / 1505 / 1511 / 1494, Xeon 943 - / 980 / 999 / 984 / 955 / 310 - the walk holds 256 KB. The tile against the nest at those shapes: M5 1405 against - 509 and 1493 against 525, Xeon 1064 against 86 and 999 against 85. The tile against the nest at d 480 ntok 3200 by - dot length 8 / 16 / 24 / 32 / 48 / 64 / 128: M5 346 / 578 / 730 / 862 / 1010 / 1105 / 1333 against 123 / 161 / 155 / - 351 / 254 / 452 / 514, Xeon 95 / 54 / 53 / 101 / 150 / 191 / 326 against 38 / 48 / 50 / 44 / 37 / 158 / 215 - the - tile wins at every length, so it serves every length and the nest stays as the race's arm. The conv front's im2col - and erf GELU ran in one lane (`bin/daslang -jit modules/dasLLAMA/harness/asr_stage_probe.das -- -m - Qwen3-ASR-0.6B-Q8_0.gguf --mmproj mmproj-Qwen3-ASR-0.6B-bf16.gguf -w gb1.wav -r 1 --lang auto`, debug-jit, one rep, - `DAS_TUNE_POLICY` unset, `DAS_TUNE_MANIFEST` the rig release's minted sidecar under - `performance/_rig/dasllama-bench/` on the Xeon and `performance/m5.tune.json` on the M5: the Xeon's `q3a.conv.mm` - 13293 ms before the tile, 5719 with it, 1446 with the two across lanes; its encode 15227 / 7728 / 2924 ms; the M5 - encode 287 ms throughout, its blocks on the Metal seat). The boards' CPU rows: `performance/gen_bench_records.das -- - --workload asr -o ` and `--workload tts -o kitten` on each box, the released `dasllama-bench` exe minted on - the box (`daspkg release`, the box class's backend, no policy override) at the branch's tip (ea6d942c2; kitten and - whisper rows at 5907be553 / 13c357f98, the same kernels), the references - `ARCHITECTURE_MEASUREMENT_AUDIO.md#asr-cpu-references` and `#the-tts-reference-instrument`, every cell the best of - the rig's reps (`--asr-reps` 2 a clip, `--tts-reps` 3 a sentence; the exe prints a clip's best alone, so no per-rep - spread is on record for these cells - `followup_general.md` row 241); das against the reference on gb1 (198.7 s), - total / encode ms, every `was` figure the 2026-10-22 release-17 boards entry's: Qwen3-ASR-0.6B Q8_0 - gnr 15797 / - 2917 against `llama-mtmd-cli` 21610 / 7304 (1.37x; was 26609 / 13996 against 15555 / 7076, 0.58x), epyc9r14 12784 / - 3071 against 15981 / 8051 (1.25x; was 16875 / 7257 against 15882 / 8040), graviton4 9209 / 3168 against 67958 / - 64362 (7.4x; was 16446 / 10522 against 67613); whisper large-v3-turbo - epyc9r14 10805 against 35132 (was 10892 - against 35126), graviton4 14633 against 82439 (was 14643 against 89922); whisper tiny - epyc9r14 946 against 1517 - (was 966 against 1531), graviton4 973 against 4042 (was 916 against 4064) - the whisper encoder is not on the fp32 - batch path. The kitten TTS rows re-measured in the pack's own voice (`tts:g2p20@Bella`, ms, das against `kittentts`, - audio seconds agreeing within 2%): gnr nano 8370 against 8211, mini 13022 against 88939; epyc9r14 9849 against - 11445, 13845 against 112833; graviton4 6991 against 8698, 10373 against 92066 - the generator's conv stages are not - on the fp32 batch path, so the tile is not in these walls. None of these rows is published: the record stores and - the site merge stay at release 17, the release re-mint owes the rows under `followup_general.md` row 238 - (`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`). What the Xeon's encode still holds (the same - probe run): `q3a.conv.mm` 1446 ms of a 2924 ms encode - the first conv's nine-tap GEMM, which the race reads at 95 - GFLOP/s against the wide layers' thousand, and the erf GELU, whose exp runs in double a lane; its decode, 11.5 s - over 451 steps, is the attention over the ~3000 audio tokens (all three in row 240's Done). The reference's encoder - runs bf16 x bf16 arithmetic where ours widens the file's bf16 to f32 and multiplies in f32 (the mmproj conv weights, - 2 bytes a weight more and no half-width dot); the whole bf16 tower set shares that gap. +boards.** `matmul_batch_core` runs a four-by-four tile (`f32_batch_tile`, +`ARCHITECTURE_CPU_KERNELS.md#f32-batch-tile-lattice`); `harness/f32_batch_race.das -- --reps 5` races its two constants +and the tile against the dot-a-cell nest it replaces in one process, interleaved, best of 5 (`bin/daslang -jit`, +`DAS_TUNE_POLICY` unset and no other override; GFLOP/s; the M5 Max under `DAS_TUNE_MANIFEST` naming a scratch sidecar +the tuner minted for this kernel - `f32_batch_tile` vec4_u4 - with `DASLLAMA_ALLOW_UNTUNED=1` for the rest, the Granite +Rapids box - AWS m8i, Xeon 6975P-C, 8 cores - under its rig's minted sidecar, vec16_u2). The X block at n 4320 d 480 +ntok 800: M5 1463 / 1461 / 1431 / 1448 / 1441 / 1419 and Xeon 1067 / 1075 / 1053 / 1049 / 947 / 349 at 64K / 128K / 256K +/ 512K / 1M / 4M; at n 1152 d 480 ntok 1536: M5 1431 / 1466 / 1497 / 1505 / 1511 / 1494, Xeon 943 / 980 / 999 / 984 / +955 / 310 - the walk holds 256 KB. The tile against the nest at those shapes: M5 1405 against 509 and 1493 against 525, +Xeon 1064 against 86 and 999 against 85. The tile against the nest at d 480 ntok 3200 by dot length 8 / 16 / 24 / 32 / +48 / 64 / 128: M5 346 / 578 / 730 / 862 / 1010 / 1105 / 1333 against 123 / 161 / 155 / 351 / 254 / 452 / 514, Xeon 95 / +54 / 53 / 101 / 150 / 191 / 326 against 38 / 48 / 50 / 44 / 37 / 158 / 215 - the tile wins at every length, so it serves +every length and the nest stays as the race's arm. The conv front's im2col and erf GELU ran in one lane (`bin/daslang +-jit modules/dasLLAMA/harness/asr_stage_probe.das -- -m Qwen3-ASR-0.6B-Q8_0.gguf --mmproj +mmproj-Qwen3-ASR-0.6B-bf16.gguf -w gb1.wav -r 1 --lang auto`, debug-jit, one rep, `DAS_TUNE_POLICY` unset, +`DAS_TUNE_MANIFEST` the rig release's minted sidecar under `performance/_rig/dasllama-bench/` on the Xeon and +`performance/m5.tune.json` on the M5: the Xeon's `q3a.conv.mm` 13293 ms before the tile, 5719 with it, 1446 with the two +across lanes; its encode 15227 / 7728 / 2924 ms; the M5 encode 287 ms throughout, its blocks on the Metal seat). The +boards' CPU rows: `performance/gen_bench_records.das -- --workload asr -o ` and `--workload tts -o kitten` on +each box, the released `dasllama-bench` exe minted on the box (`daspkg release`, the box class's backend, no policy +override) at the branch's tip (ea6d942c2; kitten and whisper rows at 5907be553 / 13c357f98, the same kernels), the +references `ARCHITECTURE_MEASUREMENT_AUDIO.md#asr-cpu-references` and `#the-tts-reference-instrument`, every cell the +best of the rig's reps (`--asr-reps` 2 a clip, `--tts-reps` 3 a sentence; the exe prints a clip's best alone, so no +per-rep spread is on record for these cells - `followup_general.md` row 241); das against the reference on gb1 (198.7 +s), total / encode ms, every `was` figure the 2026-10-22 release-17 boards entry's: Qwen3-ASR-0.6B Q8_0 - gnr 15797 / +2917 against `llama-mtmd-cli` 21610 / 7304 (1.37x; was 26609 / 13996 against 15555 / 7076, 0.58x), epyc9r14 12784 / 3071 +against 15981 / 8051 (1.25x; was 16875 / 7257 against 15882 / 8040), graviton4 9209 / 3168 against 67958 / 64362 (7.4x; +was 16446 / 10522 against 67613); whisper large-v3-turbo - epyc9r14 10805 against 35132 (was 10892 against 35126), +graviton4 14633 against 82439 (was 14643 against 89922); whisper tiny - epyc9r14 946 against 1517 (was 966 against +1531), graviton4 973 against 4042 (was 916 against 4064) - the whisper encoder is not on the fp32 batch path. The kitten +TTS rows re-measured in the pack's own voice (`tts:g2p20@Bella`, ms, das against `kittentts`, audio seconds agreeing +within 2%): gnr nano 8370 against 8211, mini 13022 against 88939; epyc9r14 9849 against 11445, 13845 against 112833; +graviton4 6991 against 8698, 10373 against 92066 - the generator's conv stages are not on the fp32 batch path, so the +tile is not in these walls. None of these rows is published: the record stores and the site merge stay at release 17, +the release re-mint owes the rows under `followup_general.md` row 238 +(`ARCHITECTURE_MEASUREMENT_RELEASE.md#board-rows-release-boundary`). What the Xeon's encode still holds (the same probe +run): `q3a.conv.mm` 1446 ms of a 2924 ms encode - the first conv's nine-tap GEMM, which the race reads at 95 GFLOP/s +against the wide layers' thousand, and the erf GELU, whose exp runs in double a lane; its decode, 11.5 s over 451 steps, +is the attention over the ~3000 audio tokens (the GELU is the float kernel `gelu_erf` of the 2026-10-25 entry, the +attention row 242, the conv front row 240's operand item). The reference's encoder runs bf16 x bf16 arithmetic where +ours widens the file's bf16 to f32 and multiplies in f32 (the mmproj conv weights, 2 bytes a weight more and no +half-width dot); the whole bf16 tower set shares that gap. - **MEASURED (2026-10-24, `direction-grade`) - the Granite Rapids board's losses are the DeltaNet scan and the f32 conv front, not the AMX crowns.** AWS m8i.4xlarge (Xeon 6975P-C, 8 cores with SMT, 61 GB, the `x86-amx` class), diff --git a/modules/dasLLAMA/REVIEW_AUDIO.md b/modules/dasLLAMA/REVIEW_AUDIO.md index 123916f218..a1d8acb696 100644 --- a/modules/dasLLAMA/REVIEW_AUDIO.md +++ b/modules/dasLLAMA/REVIEW_AUDIO.md @@ -22,9 +22,10 @@ goes in the family file.** **A GEMM against model weights in an ASR family file that does not go through a `*_mm*` wrapper or `mm_blob_b` is a defect, hand-written dot-product loops included.** -**A matrix product of two activations - tensors the encoder or decoder computes, never a weight -or the mel frontend's transform table (`build_dft_twiddles`) - in an ASR family file that does -not go through `gemm_f32_jo` is a defect, hand-written loops included.** +**A matrix product of two activations - tensors the encoder or decoder computes, never a weight or +a constant table built from the audio config alone (the window, the DFT twiddles, the mel +filterbank) - in an ASR family file that does not go through `gemm_f32_jo` is a defect, +hand-written loops included.** **A buffer reused across encodes in an audio-rail file that is not `@scratch` - on its declaration, or on the parameter of every callee that grows it - is a defect.** diff --git a/modules/dasLLAMA/REVIEW_KQ_FORMATS.md b/modules/dasLLAMA/REVIEW_KQ_FORMATS.md index 4e0bfc59f9..e790018457 100644 --- a/modules/dasLLAMA/REVIEW_KQ_FORMATS.md +++ b/modules/dasLLAMA/REVIEW_KQ_FORMATS.md @@ -100,16 +100,15 @@ every lever at its default, a site that backend's serving call reaches reads the converted copy or hands a GPU site to the CPU's kernels is a defect - state that site as a gap instead, or cite the follow-up row that ledgers it.** -**A diff that states, in any document, PR description, test name or predicate, that a backend -serves a weight format at a site lands or cites a subtest that loads a file whose plane at that -site is in that format on disk, and asserts that the loader's format tag for that plane equals -its disk type and that a kernel counter (`metal_kernel_coverage_of(name)` on Metal, -`vk_kernel_coverage_of(name)` on Vulkan, `cud_kernel_coverage_of(name)` on CUDA) that no other -plane of that file moves rose during the run.** The file is a downloaded -`performance/model_specs.das` row's file, or, when no downloaded row carries the format, a row -whose `file` ends `-local.gguf` and whose `recipe` requantizes into it. A subtest that only -asserts the backend ran still passes when the site reads a converted copy or runs on the CPU -pre-step. +**A diff that states, in any document, PR description, test name or predicate, that a GPU backend +serves a `KqFmt` member at a site lands or cites a subtest that loads a file whose plane at that +site is in that format on disk, and asserts that the loader's format tag for that plane equals its +disk type and that a kernel counter (`metal_kernel_coverage_of(name)` on Metal, +`vk_kernel_coverage_of(name)` on Vulkan, `cud_kernel_coverage_of(name)` on CUDA) that no other plane +of that file moves rose during the run.** The file is a downloaded `performance/model_specs.das` +row's file, or, when no downloaded row carries the format, a row whose `file` ends `-local.gguf` and +whose `recipe` requantizes into it. A subtest that only asserts the backend ran still passes when +the site reads a converted copy or runs on the CPU pre-step. **A diff that adds a lever, off by default, that makes a site read a weight format through a converted copy names the lever in the same change - in `ARCHITECTURE_GPU_ASYMMETRIES.md` when one diff --git a/modules/dasLLAMA/REVIEW_VISION.md b/modules/dasLLAMA/REVIEW_VISION.md index 3322ddbcfd..a9967e4fbb 100644 --- a/modules/dasLLAMA/REVIEW_VISION.md +++ b/modules/dasLLAMA/REVIEW_VISION.md @@ -30,16 +30,16 @@ does not say so, is a defect.** **A diff that puts a vision family's `set_*` runtime knob, or the module global it writes, anywhere but that family's file is a defect.** -**A family gaining support for a media kind adds that kind's span markers to that family's chat -template, never to another family's; a diff claiming that support while the family's chat template -or vocab lacks the markers is a defect.** Span markers are the template text that opens and closes -the media rows. +**A decoder family (`dasllama/dasllama_arch_.das`) gaining support for a media kind adds +that kind's span markers to that family's chat template, never to another family's; a diff claiming +that support while the family's chat template or vocab lacks the markers is a defect.** Span markers +are the template text that opens and closes the media rows. -**A media splice's rows reaching the layer stack as more than one `forward_prefill_embd` call, or -as row chunks inside one call, is a defect; a driver declining the whole call to the CPU loop, or -splitting its command buffers per layer, still makes one call, and that call may also carry the -text tokens before and after the span** - the span bounds are counted from the start of the call, so a boundary inside the span -changes the mask. +**A media splice's rows reaching the layer stack as more than one `forward_prefill_embd` call, or as +row chunks inside one call, is a defect; a driver declining the whole call to the CPU loop, or +splitting its command buffers per layer, still makes one call, and that call may also carry the text +tokens before and after the span** - the span bounds are counted from the start of the call, so a +boundary inside the span changes the mask. **A diff that adds or changes a session kind or an admission mode of the in-process scheduler (`dasllama/dasllama_scheduler.das`) that a stream carrying decoded media can reach either serves @@ -54,6 +54,6 @@ is what keeps BPE merges from crossing the media. request carries in `media_embd` - also updates, in the same change, every file that fills or reads a `media_embd`.** -**A family's new arm for a media kind that reaches the layer stack by any path but -`forward_prefill_embd` is a defect, a second prefill body for it included** - write a sibling +**A vision or ASR family file's new arm for a media kind that reaches the layer stack by any path +but `forward_prefill_embd` is a defect, a second prefill body for it included** - write a sibling `eval_embd_span*` entry that feeds the same body instead. diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 4e08e1e857..aae3cde168 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -4015,7 +4015,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k } // ===== the bf16-weight batch family: a native bf16 plane (a tower's or the gemma-E projection's) against -// f32 activations, through the AMX bf16 tile - the activations rounded to bf16 as the reference's are +// f32 activations, through the AMX bf16 tile - the activations rounded to bf16 first //! the vector contract's reference values: no tile form, four tokens a call def bf16w_tokstep_gen() : int => 4 diff --git a/modules/dasLLAMA/dasllama/dasllama_qwen3a.das b/modules/dasLLAMA/dasllama/dasllama_qwen3a.das index 220b486f07..097754c2d7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_qwen3a.das +++ b/modules/dasLLAMA/dasllama/dasllama_qwen3a.das @@ -439,7 +439,7 @@ def log_mel_qwen3a(samples : array | #; n_mel : int64) : WhisperMel { } } // power -> mel: float products in groups of 4 summed in float, groups accumulated in - // double (reference-exact rounding), floor 2^-24, log10 + // double (the rounding the parity cells hold), floor 2^-24, log10 let mel_floor = 5.960464477539063e-08lf let log10_e = 0.43429448190325176lf maybe_parallel_for(0, int(n_len), lanes_for_work(n_len * n_mel * n_bins, 0)) $(rb, re) { diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index c0acd11511..152cb077a6 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -2296,18 +2296,22 @@ `--ref-clean` naming the clean-cpu `llama-bench` `benchmarks/setup_lcpp_ref.das` builds at `DEFAULT_REF_SHA`. 240. **The bf16 towers' conv fronts are F16, and the half-width family has only its AMX leg.** `bf16w_tile_gen` - (`dasllama_math_gen.das`) multiplies a native bf16 plane as the reference does - the activations rounded to bf16, - the TMUL bf16 tile over a k-pair panel - and `matmul_bf16_batch` reaches it where a sidecar names an `amx_bf16` - perm; the gemma-E per-layer projection and the bf16 vision patch planes ride it (the 2026-10-25 `PERF_LEDGER.md` - Granite Rapids entry, `debug-jit`: E4B pp512 359.8 -> 368.8 tok/s over the widen walk against the committed - `records/gnr.json` E4B CPU pp512 row's 325.7, the tile 3921 against the f32 plane's 782 GFLOP/s at that shape). What - is left: the Qwen3-ASR mmproj's conv weights are F16 (`a.conv2d.*.weight`, `a.conv_out.weight`), its blocks run the - q8 lane, so the conv front's 1123 ms of f32 tile on the Xeon reaches the matrix unit only through an f16 operand - - AMX-FP16 (Granite Rapids on, Sapphire Rapids off) or an f16-to-bf16 panel (three mantissa bits under the reference's - f16 x f16); and the family has no vector leg, so Zen4 (`vdpbf16ps`, avx512bf16) and Graviton (`bfdot` / `bfmmla`, - the arm bf16 extension) run the widen walk. Done = the conv front's operand ruled and served through the family, - `vdpbf16ps` and `bfdot` perms in `dasllama_gemm_gen.das` over the same panel with the probe gating them, and the - Qwen3-ASR, gemma-4-E4B and image-chat CPU rows re-minted on every board box at the release re-mint. + (`dasllama_math_gen.das`) multiplies a native bf16 plane as bf16 times bf16 - the activations rounded to bf16, the + TMUL bf16 tile over a k-pair panel - and `matmul_bf16_batch` reaches it where a sidecar names an `amx_bf16` perm; + the gemma-E per-layer projection and the bf16 vision patch planes ride it (the 2026-10-25 `PERF_LEDGER.md` Granite + Rapids entry, `debug-jit`: E4B pp512 359.8 -> 368.8 tok/s over the widen walk against the committed + `records/gnr.json` E4B CPU pp512 row's 325.7 (the 2026-10-22 release-17 entry), the tile 3921 against the f32 + plane's 782 GFLOP/s at that shape). What is left: the Qwen3-ASR mmproj's conv weights are F16 (`a.conv2d.*.weight`, + `a.conv_out.weight`), its blocks run the q8 lane, so the conv front's 1123 ms of f32 tile on the Xeon reaches the + matrix unit only through an f16 operand - AMX-FP16 (Granite Rapids on, Sapphire Rapids off) or an f16-to-bf16 panel + (three mantissa bits under the reference's f16 x f16); and the family has no vector leg, so Zen4 (`vdpbf16ps`, + avx512bf16) and Graviton (`bfdot` / `bfmmla`, the arm bf16 extension) run the widen walk. Done = the conv front's + operand ruled and served through the family, `vdpbf16ps` and `bfdot` perms in `dasllama_gemm_gen.das` over the same + panel with the probe gating them, and the Qwen3-ASR, gemma-4-E4B and image-chat CPU rows re-minted on every board + box at the release re-mint (`performance/gen_bench_records.das -- --workload asr -o Qwen3-ASR` and `--workload + llm,image -o gemma-4-E4B-it-Q8_0` with the release's rig exe; the references `llama-bench` clean-cpu and stock + builds at `benchmarks/setup_lcpp_ref.das`'s `DEFAULT_REF_SHA` through `LLAMA_BENCH_CLEAN` / `LLAMA_BENCH_STOCK`, + `llama-mtmd-cli` beside the stock build; `ARCHITECTURE_MEASUREMENT_AUDIO.md#asr-cpu-references` for the ASR pair). 241. **The bench exe's ASR and TTS cells print a clip's best of their reps alone.** `lcpp_bench --asr` and `--tts` keep the minimum over `-r` reps (`r.enc_ms = min(...)`, the sentence sums) and the record row carries that one wall, @@ -2326,4 +2330,7 @@ against our 24.6 (11115 ms over 451). Done = a ruling on the ASR session's CPU codec - q8_0 by default with the parity cells' bars re-cut on its transcripts, or f16 kept and the row closed - written into `ARCHITECTURE_MEASUREMENT_AUDIO.md` and the facade's default, and the Qwen3-ASR CPU rows re-minted on every board - box at the release re-mint. \ No newline at end of file + box at the release re-mint (`performance/gen_bench_records.das -- --workload asr -o Qwen3-ASR` with the release's + rig exe; the references `llama-bench` clean-cpu and stock builds at `benchmarks/setup_lcpp_ref.das`'s + `DEFAULT_REF_SHA` through `LLAMA_BENCH_CLEAN` / `LLAMA_BENCH_STOCK`, `llama-mtmd-cli` beside the stock build; + `ARCHITECTURE_MEASUREMENT_AUDIO.md#asr-cpu-references` for the ASR pair).