Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -1232,6 +1232,17 @@ Against NumPy 2.5.4 (OpenBLAS, `OPENBLAS_NUM_THREADS=1`, the same core):
| N1 float32 | 30.4 / 16.2 µs | 1.19 / 0.94 ms | 68.2 / 60.6 ms |
| N1 float64 | 48.2 / 31.3 µs | 2.46 / 1.95 ms | 154 / 123 ms |

v0.26.1 packs full B panels with one fixed-size array move per row (8 or 16
elements, lowered to vector moves), where `packB` had copied element by
element. Profiling had put it at 21% of a 64² float32 product.

| `MatMul` vs v0.26.0 (`abbench`, 10 rounds) | 64² | 256² | 1024² |
|---|---:|---:|---:|
| Zen 3, float32 | −11.6% | −7.1% | −1.7% |
| Zen 3, float64 | −4.4% | −2.3% | ~ |
| Neoverse N1, float32 | −16.4% | −8.9% | ~ |
| Neoverse N1, float64 | −9.5% | ~ | ~ |

**Open defect: products are still slower than OpenBLAS on one core**, by
1.1–1.3× at 1024² and up to 1.9× at 64². The gap is widest where the work
around the kernel weighs most: packing, edge tiles, and the result's
Expand Down
24 changes: 24 additions & 0 deletions internal/kernels/gemm.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,9 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) {
nr := min(NR, nc-jr)
dst := pb[jr*kc:]
if b.CS == 1 {
if nr == NR && packRowsWhole(dst, b.Data, b.Off+pc*b.RS+jc+jr, b.RS, kc, NR) {
continue
}
// Element loops, not copy and clear: on runs this short (NR
// elements) each of those is a runtime call.
for p := 0; p < kc; p++ {
Expand Down Expand Up @@ -232,6 +235,27 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) {
}
}

// packRowsWhole copies kc runs of nr elements, the p-th starting at
// src[s+p*rs], to dst[p*nr:], each as one fixed-size array assignment, which
// the compiler lowers to a few vector moves instead of a loop or a runtime
// call. It reports false, copying nothing, for a width it has no array type
// for; the caller then copies element by element.
func packRowsWhole[T gemmFloat](dst, src []T, s, rs, kc, nr int) bool {
switch nr {
case 8:
for p := 0; p < kc; p++ {
*(*[8]T)(dst[p*8:]) = *(*[8]T)(src[s+p*rs:])
}
case 16:
for p := 0; p < kc; p++ {
*(*[16]T)(dst[p*16:]) = *(*[16]T)(src[s+p*rs:])
}
default:
return false
}
return true
}

// packA copies A[ic:ic+mc][pc:pc+kc] into MR-tall row panels: for each panel ir
// the layout is pa[ir*kc + p*MR + r] = A(ic+ir+r, pc+p), with the r >= mr rows
// of an edge panel zero-filled so the micro-kernel reads a full MR.
Expand Down
27 changes: 27 additions & 0 deletions internal/kernels/gemm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,33 @@ func TestGemmFloatTolerance(t *testing.T) {
}
}

// TestPackRowsWhole: the whole-run copy moves exactly kc runs of 8 or 16
// strided elements, and declines any other width without writing.
func TestPackRowsWhole(t *testing.T) {
src := randVec(200, 3)
for _, nr := range []int{8, 16} {
const s, rs, kc = 5, 21, 7
dst := make([]float64, kc*nr+1)
if !packRowsWhole(dst, src, s, rs, kc, nr) {
t.Fatalf("nr=%d declined", nr)
}
for p := 0; p < kc; p++ {
for c := 0; c < nr; c++ {
if dst[p*nr+c] != src[s+p*rs+c] {
t.Fatalf("nr=%d [%d][%d] = %v, want %v", nr, p, c, dst[p*nr+c], src[s+p*rs+c])
}
}
}
if dst[kc*nr] != 0 {
t.Fatalf("nr=%d wrote past its %d runs", nr, kc)
}
}
dst := make([]float64, 40)
if packRowsWhole(dst, src, 0, 10, 4, 5) || dst[0] != 0 {
t.Fatal("nr=5 copied")
}
}

// TestGemmBufferRealloc forces block sizes larger than the pooled buffers were
// first sized for, exercising the reallocation branch in getPackBuf, then a
// normal-sized call to confirm the pool still serves valid scratch.
Expand Down
Loading