From 7ff1b3426d074ceba32c664ca44759628fe1bb0b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 26 Oct 2026 21:20:41 +0100 Subject: [PATCH 1/2] perf(linalg): native float32 GEMM on amd64 (FMA) and arm64 The packed GEMM driver is now generic over float32 and float64. Each element type has its own micro-kernel, tile and pack-buffer pools (gemmKernel[T]). float64 keeps its kernels and MR/NR. float32 gets: - arm64: gemmMicro8x8S, NEON 8x8, by-element FMLA .4S (WORD-encoded, checked against go tool objdump); - amd64: gemmMicro6x16FMA, AVX2/FMA 6x16 (the BLIS Haswell sgemm shape), only when the CPU has FMA. Float32 MatMul (and Dot, which goes through it) runs on that kernel and accumulates in float32, as NumPy's BLAS sgemm does. Without a kernel (no FMA, or any other architecture) it still widens to the float64 GEMM. gemmEdge's scratch tile now lives at the end of the worker's A buffer. The kernel is called through a func value, so a stack tile would escape to the heap at every edge tile. Tests: the float32 GEMM bit-exact against a float32 ikj oracle on integer data (every block configuration, a transposed and offset B view, serial and parallel), within k*eps32*sum|ab| on random data, NaN/Inf reaching only their row, a fence test of the kernel's reach, and the root MatMul on a transposed view and a mixed Int8 operand on both paths. Co-Authored-By: Claude Opus 5.5 --- benchmarks/dtype_bench_test.go | 4 +- internal/kernels/asmgen/amd64/gen.go | 74 +++++++++++ internal/kernels/asmgen/arm64/gen.go | 74 ++++++++++- internal/kernels/fence_test.go | 20 +++ internal/kernels/gemm.go | 181 +++++++++++++++++---------- internal/kernels/gemm32_amd64.go | 24 ++++ internal/kernels/gemm32_arm64.go | 16 +++ internal/kernels/gemm32_other.go | 7 ++ internal/kernels/gemm32_test.go | 149 ++++++++++++++++++++++ internal/kernels/gemm_amd64_test.go | 11 ++ internal/kernels/sum_amd64.s | 142 +++++++++++++++++++++ internal/kernels/sum_arm64.s | 87 +++++++++++++ linalg.go | 38 +++++- linalg_test.go | 37 ++++++ 14 files changed, 787 insertions(+), 77 deletions(-) create mode 100644 internal/kernels/gemm32_amd64.go create mode 100644 internal/kernels/gemm32_arm64.go create mode 100644 internal/kernels/gemm32_other.go create mode 100644 internal/kernels/gemm32_test.go diff --git a/benchmarks/dtype_bench_test.go b/benchmarks/dtype_bench_test.go index c431e11..49d4de2 100644 --- a/benchmarks/dtype_bench_test.go +++ b/benchmarks/dtype_bench_test.go @@ -73,8 +73,8 @@ func BenchmarkDTypeMax(b *testing.B) { // BenchmarkDTypeMatMul is a square product, 2·n³ flops per op. func BenchmarkDTypeMatMul(b *testing.B) { for _, dt := range []nd.DType{nd.Float64, nd.Float32} { - for _, n := range []int{64, 256} { - x, err := vec(n * n).AsType(dt).Reshape(n, n) + for _, n := range []int{64, 256, 1024} { + x, err := vec(n*n).AsType(dt).Reshape(n, n) if err != nil { b.Fatal(err) } diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index dda3fc6..ba9d65b 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -46,6 +46,7 @@ func main() { f.Add(binKernel("mulSSE2", "MULPD", "MULSD")) f.Add(binKernel("divSSE2", "DIVPD", "DIVSD")) f.Add(gemmFMAKernel()) + f.Add(gemm32FMAKernel()) f.Add(dotFMAKernel()) f.Add(gemmSSE2HalfKernel()) f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) @@ -487,6 +488,79 @@ func gemmFMAKernel() *emit.Function { return b.Func() } +// gemm32NR is the float32 tile width on amd64: two YMM of eight floats. +const gemm32NR = 16 + +// gemm32FMAKernel builds gemmMicro6x16FMA(kc int, pa, pb, c *float32, ldc int), +// the float32 micro-kernel of the packed GEMM: +// +// for p in [0,kc): C[r][col] += pa[p*6+r] * pb[p*16+col] (r<6, col<16) +// +// gemmFMAKernel at twice the lanes, the BLIS Haswell sgemm shape: the 16-float +// B row in Y12,Y13, each of the six A values broadcast with VBROADCASTSS, and +// twelve VFMADD231PS into Y0..Y11 (row r in Y(2r), Y(2r+1)). Gated on hasFMA. +func gemm32FMAKernel() *emit.Function { + b := amd64.NewFunc("gemmMicro6x16FMA", gemmSig(), 0) + b.LoadArg("kc", "CX") + b.LoadArg("pa", "SI") + b.LoadArg("pb", "DI") + b.LoadArg("c", "DX") + b.LoadArg("ldc", "R8") + b.Raw("SHLQ $2, R8") // ldc -> bytes + // Prefetch the C tile: six rows of 64 bytes, each on one or two lines. + b.Raw("MOVQ DX, R9") + for r := 0; r < gemmMR; r++ { + b.Raw("PREFETCHT0 (R9)") + b.Raw("PREFETCHT0 63(R9)") + if r < gemmMR-1 { + b.Raw("ADDQ R8, R9") + } + } + for r := 0; r < 12; r++ { + b.Raw(fmt.Sprintf("VXORPS Y%d, Y%d, Y%d", r, r, r)) + } + b.Raw("MOVQ CX, R10") + b.Raw("SHRQ $1, R10") + b.Raw("JZ stail") + b.Raw("skloop:") + b.Raw(fmt.Sprintf("PREFETCHT0 %d(SI)", 16*gemmMR*4)) + fma32Step(b, 0) + fma32Step(b, 1) + b.Raw(fmt.Sprintf("ADDQ $%d, SI", 2*4*gemmMR)) + b.Raw(fmt.Sprintf("ADDQ $%d, DI", 2*4*gemm32NR)) + b.Raw("DECQ R10") + b.Raw("JNZ skloop") + b.Raw("stail:") + b.Raw("TESTQ $1, CX") + b.Raw("JZ sstore") + fma32Step(b, 0) + b.Raw("sstore:") + for r := 0; r < gemmMR; r++ { + b.Raw(fmt.Sprintf("VADDPS (DX), Y%d, Y15", 2*r)) + b.Raw("VMOVUPS Y15, (DX)") + b.Raw(fmt.Sprintf("VADDPS 32(DX), Y%d, Y15", 2*r+1)) + b.Raw("VMOVUPS Y15, 32(DX)") + if r < gemmMR-1 { + b.Raw("ADDQ R8, DX") + } + } + b.Raw("VZEROUPPER") + b.Ret() + return b.Func() +} + +// fma32Step emits one k step of the 6x16 float32 FMA tile. +func fma32Step(b *amd64.Builder, step int) { + ao, bo := 4*gemmMR*step, 4*gemm32NR*step + b.Raw(fmt.Sprintf("VMOVUPS %d(DI), Y12", bo)) // B cols 0..7 + b.Raw(fmt.Sprintf("VMOVUPS %d(DI), Y13", bo+32)) // B cols 8..15 + for r := 0; r < gemmMR; r++ { + b.Raw(fmt.Sprintf("VBROADCASTSS %d(SI), Y14", ao+4*r)) + b.Raw(fmt.Sprintf("VFMADD231PS Y12, Y14, Y%d", 2*r)) + b.Raw(fmt.Sprintf("VFMADD231PS Y13, Y14, Y%d", 2*r+1)) + } +} + // dotFMAKernel builds dotFMA(a, b *float64, n int) float64, the inner product // with four YMM accumulators (16 doubles per iteration, one VFMADD231PD per 4 // doubles reading b straight from memory): two loads per FMA make the loop diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index d18d8a7..ab69ed8 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -57,6 +57,7 @@ func main() { f.Add(extremum32Kernel("max32NEON", "VFMAX", "FMAXS")) f.Add(extremum32Kernel("min32NEON", "VFMIN", "FMINS")) f.Add(gemmKernel()) + f.Add(gemm32Kernel()) f.Add(blocks8Kernel()) if err := os.WriteFile("sum_arm64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) @@ -290,9 +291,9 @@ func extremum32Kernel(name, vop, sop string) *emit.Function { } b.Raw("SUB $16, R1").Raw("B block"). Raw("fold:"). - Raw(vop+" V1.S4, V0.S4, V0.S4"). - Raw(vop+" V3.S4, V2.S4, V2.S4"). - Raw(vop+" V2.S4, V0.S4, V0.S4") + Raw(vop + " V1.S4, V0.S4, V0.S4"). + Raw(vop + " V3.S4, V2.S4, V2.S4"). + Raw(vop + " V2.S4, V0.S4, V0.S4") for l := 0; l < 4; l++ { b.Raw(fmt.Sprintf("VMOV V0.S[%d], R%d", l, 3+l)) } @@ -470,6 +471,73 @@ func gemmKernel() *emit.Function { return b.Func() } +// fmlaElemS is fmlaElem for single precision: +// +// FMLA V{acc}.4S, V{breg}.4S, V{areg}.S[idx] => V{acc} += V{breg} * V{areg}[idx] +// +// The same Advanced SIMD "FMLA (by element)" encoding with sz=0 (ARM C7-2): +// +// 0|1|0|01111|1|0|L|M|Rm[3:0]|0001|H|0|Rn|Rd +// +// where the element index is H:L and, for sz=0, the register is M:Rm, so +// areg may be any of V0..V31. idx in {0,1,2,3}. +func fmlaElemS(acc, breg, areg, idx int) string { + const base = 0x4F801000 // Q=1, sz=0, opcode 0001: FMLA by element .4S + M := (areg >> 4) & 1 + Rm := areg & 0xF + H, L := (idx>>1)&1, idx&1 + word := base | (L << 21) | (M << 20) | (Rm << 16) | (H << 11) | (breg << 5) | acc + return fmt.Sprintf("WORD $0x%08X // FMLA V%d.4S, V%d.4S, V%d.S[%d]", + word, acc, breg, areg, idx) +} + +// gemm32Kernel builds gemmMicro8x8S(kc int, pa, pb, c *float32, ldc int), the +// float32 micro-kernel of the packed GEMM: an 8-row x 8-col tile, +// +// for p in [0,kc): C[r][col] += pa[p*8+r] * pb[p*8+col] (r<8, col<8) +// +// the float64 kernel's structure at twice the lanes. The tile lives in V0..V15 +// (row r in V(2r), V(2r+1), four floats each), the B row in V16,V17 and the +// eight A values in V18=[a0..a3], V19=[a4..a7], which the by-element FMLA +// (fmlaElemS) reads lane by lane. Sixteen independent accumulator chains cover +// the FMA latency of two 4-cycle pipes (Neoverse N1) with room to spare; each +// k step is two loads for sixteen FMAs. The closing C += acc is a VFADD. +func gemm32Kernel() *emit.Function { + b := arm64.NewFunc("gemmMicro8x8S", gemmSig(), 0) + b.LoadArg("kc", "R0"). + LoadArg("pa", "R1"). + LoadArg("pb", "R2"). + LoadArg("c", "R3"). + LoadArg("ldc", "R4"). + Raw("LSL $2, R4, R4") // ldc -> bytes + for r := 0; r < 16; r++ { + b.Raw(fmt.Sprintf("VEOR V%d.B16, V%d.B16, V%d.B16", r, r, r)) + } + b.Raw("CBZ R0, sstore"). + Raw("skloop:"). + Raw("VLD1.P 32(R2), [V16.S4, V17.S4]"). // B row: 8 floats + Raw("VLD1.P 32(R1), [V18.S4, V19.S4]") // A column: 8 floats + for r := 0; r < 8; r++ { + areg, lane := 18+r/4, r%4 + b.Raw(fmlaElemS(2*r, 16, areg, lane)) + b.Raw(fmlaElemS(2*r+1, 17, areg, lane)) + } + b.Raw("SUB $1, R0"). + Raw("CBNZ R0, skloop"). + Raw("sstore:") + for r := 0; r < 8; r++ { + b.Raw("VLD1 (R3), [V16.S4, V17.S4]") + b.Raw(fmt.Sprintf("VFADD V%d.S4, V16.S4, V16.S4", 2*r)) + b.Raw(fmt.Sprintf("VFADD V%d.S4, V17.S4, V17.S4", 2*r+1)) + b.Raw("VST1 [V16.S4, V17.S4], (R3)") + if r < 7 { + b.Raw("ADD R4, R3, R3") + } + } + b.Ret() + return b.Func() +} + // sumKernel builds sumNEON(a *float64, n int) float64. // // The lane-parallel accumulation is VFMLA against a vector of 1.0: acc += src * diff --git a/internal/kernels/fence_test.go b/internal/kernels/fence_test.go index 2ed7217..3ec20e9 100644 --- a/internal/kernels/fence_test.go +++ b/internal/kernels/fence_test.go @@ -135,3 +135,23 @@ func TestGemmMicroStaysInsideItsOperands(t *testing.T) { run() forEachFallback(run) } + +// TestGemmMicro32StaysInsideItsOperands fences the float32 GEMM kernel's +// panels and C tile, where there is one. +func TestGemmMicro32StaysInsideItsOperands(t *testing.T) { + g := gemm32 + if g == nil { + t.Skip("no float32 GEMM kernel here") + } + for kc := 1; kc <= 9; kc++ { + for _, ldc := range []int{g.nr, g.nr + 3, 64} { + for _, pa := range fences32(t, kc*g.mr) { + for _, pb := range fences32(t, kc*g.nr) { + for _, dst := range fences32(t, (g.mr-1)*ldc+g.nr) { + g.micro(kc, pa, pb, dst, ldc) + } + } + } + } + } +} diff --git a/internal/kernels/gemm.go b/internal/kernels/gemm.go index 18b585b..7326284 100644 --- a/internal/kernels/gemm.go +++ b/internal/kernels/gemm.go @@ -20,6 +20,11 @@ import ( // (gemmMicro, arch-specific: NEON 4x8 on arm64, AVX2/FMA 6x8 or SSE2 on amd64, // scalar 4x4 elsewhere) does the inner work. Parallelised across row bands. // +// The driver is generic over the element type: float64 everywhere (gemm64), and +// float32 where a float32 micro-kernel exists (gemm32: NEON 8x8 on arm64, +// AVX2/FMA 6x16 on amd64), which accumulates in float32 as a BLAS sgemm does. +// Each element type brings its own tile and its own pack-buffer pools. +// // The arithmetic is the ikj accumulation dst[i][j] += a[i][p]*b[p][j]: within // one KC block the products are summed in p order, and each block's sum is then // added into dst. For k <= KC that is the scalar oracle's order exactly; above @@ -48,28 +53,51 @@ var ( // goroutine-loop overhead stays amortized. A var so it can be tuned/pinned. var bandTilesMR = 8 -// Mat is a read-only strided view of a matrix: element (i, j) is +// gemmFloat is the element types the packed GEMM multiplies. +type gemmFloat interface{ float32 | float64 } + +// MatOf is a read-only strided view of a matrix: element (i, j) is // Data[Off + i*RS + j*CS]. Packing reads the operands through it, so a // transposed, sliced or offset view is multiplied in place — the packing copy // the GEMM makes anyway does the gather — instead of being materialised into // a contiguous copy first (what BLAS's trans flags buy). Strides may be // negative or zero; every (i, j) in range must index Data. -type Mat struct { - Data []float64 +type MatOf[T gemmFloat] struct { + Data []T Off, RS, CS int } +// Mat and Mat32 are the float64 and float32 operand views. +type ( + Mat = MatOf[float64] + Mat32 = MatOf[float32] +) + // rowMajor views a contiguous row-major matrix with ld elements per row. -func rowMajor(data []float64, ld int) Mat { return Mat{Data: data, RS: ld, CS: 1} } +func rowMajor[T gemmFloat](data []T, ld int) MatOf[T] { + return MatOf[T]{Data: data, RS: ld, CS: 1} +} + +// gemmKernel is one element type's micro-kernel, the mr x nr tile it computes, +// and the pools of pack buffers sized for that tile. micro adds the mr x nr tile +// sum_p pa[p*mr..]*pb[p*nr..] into dst, whose rows are ldc apart; see gemmMicro. +type gemmKernel[T gemmFloat] struct { + mr, nr int + micro func(kc int, pa, pb, dst []T, ldc int) + paPool, pbPool sync.Pool +} + +// gemm64 is the float64 GEMM, on the per-architecture gemmMicro. +var gemm64 = &gemmKernel[float64]{mr: MR, nr: NR, micro: gemmMicro} // packBuf is a pair of pack buffers: pa for the MC*KC A panel, pb for the KC*NC // B panel. The two come from separate pools, because the parallel GEMM needs // one pa per worker but only two pb in all: drawing a pair per worker held // 16 MiB of unused B buffers at 16 workers, which the garbage collector empties // out of the pool and the next call zeroes again. -type packBuf struct { - pa []float64 - pb []float64 +type packBuf[T gemmFloat] struct { + pa []T + pb []T } // roundUp returns x rounded up to the next multiple of m (m >= 1). @@ -82,32 +110,37 @@ func roundUp(x, m int) int { return (x + m - 1) / m * m } // whose height is not a multiple of MR would write past the buffer. Likewise the // B buffer rounds the column extent up to a multiple of NR. (With the old MR=4 // and MC=256 the rounding was a no-op since 4 | 256; MR=6 makes it load-bearing.) -func paCap() int { return roundUp(blockMC, MR) * blockKC } -func pbCap() int { return blockKC * roundUp(blockNC, NR) } - -var paPool, pbPool sync.Pool +// +// The A buffer also carries, past the panels, the mr x nr scratch tile gemmEdge +// runs the kernel into. The tile cannot live on gemmEdge's stack: the kernel is +// called through a func value, so escape analysis would move it to the heap at +// every edge tile. +func (g *gemmKernel[T]) paCap() int { return roundUp(blockMC, g.mr)*blockKC + g.mr*g.nr } +func (g *gemmKernel[T]) pbCap() int { return blockKC * roundUp(blockNC, g.nr) } // getPA and getPB return a pooled pack buffer sized for the current block // parameters, allocating one when the pool is empty or its buffer is too small // (a test may raise blockMC/KC/NC). putPA and putPB return it. -func getPA() []float64 { return getScratch(&paPool, paCap()) } -func getPB() []float64 { return getScratch(&pbPool, pbCap()) } -func putPA(b []float64) { paPool.Put(&b) } -func putPB(b []float64) { pbPool.Put(&b) } +func (g *gemmKernel[T]) getPA() []T { return getScratch[T](&g.paPool, g.paCap()) } +func (g *gemmKernel[T]) getPB() []T { return getScratch[T](&g.pbPool, g.pbCap()) } +func (g *gemmKernel[T]) putPA(b []T) { g.paPool.Put(&b) } +func (g *gemmKernel[T]) putPB(b []T) { g.pbPool.Put(&b) } -func getScratch(p *sync.Pool, n int) []float64 { - if b, ok := p.Get().(*[]float64); ok && cap(*b) >= n { +func getScratch[T gemmFloat](p *sync.Pool, n int) []T { + if b, ok := p.Get().(*[]T); ok && cap(*b) >= n { return (*b)[:n] } - return make([]float64, n) + return make([]T, n) } // getPackBuf returns a pooled pair, for the single-worker paths. -func getPackBuf() *packBuf { return &packBuf{pa: getPA(), pb: getPB()} } +func (g *gemmKernel[T]) getPackBuf() *packBuf[T] { + return &packBuf[T]{pa: g.getPA(), pb: g.getPB()} +} -func putPackBuf(b *packBuf) { - putPA(b.pa) - putPB(b.pb) +func (g *gemmKernel[T]) putPackBuf(b *packBuf[T]) { + g.putPA(b.pa) + g.putPB(b.pb) } // packGemmRows computes the row band [r0,r1) of dst = a(m x k) * b(k x n) with @@ -117,14 +150,14 @@ func putPackBuf(b *packBuf) { // r0 (the parallel splitter guarantees this) so packed A panels never straddle a // band boundary; r1 may be m. This is the single-worker entry: it packs its own // B panel. The multi-worker path (gemmBand) shares one packed B across workers. -func packGemmRows(r0, r1 int, dst []float64, a, b Mat, k, n int, buf *packBuf) { +func (g *gemmKernel[T]) packGemmRows(r0, r1 int, dst []T, a, b MatOf[T], k, n int, buf *packBuf[T]) { pa, pb := buf.pa, buf.pb for jc := 0; jc < n; jc += blockNC { // L3 column block of B nc := min(blockNC, n-jc) for pc := 0; pc < k; pc += blockKC { // L2 contraction block kc := min(blockKC, k-pc) - packB(b, pc, kc, jc, nc, pb) // pack B(kc x nc) -> NR-wide panels - gemmBand(r0, r1, jc, nc, pc, kc, dst, a, n, pa, pb) + g.packB(b, pc, kc, jc, nc, pb) // pack B(kc x nc) -> NR-wide panels + g.gemmBand(r0, r1, jc, nc, pc, kc, dst, a, n, pa, pb) } } } @@ -138,19 +171,21 @@ func packGemmRows(r0, r1 int, dst []float64, a, b Mat, k, n int, buf *packBuf) { // is what lets several workers share one packed B: the redundant full-B repack // each worker used to do — 1 MiB copied GOMAXPROCS times per kc block — was the // dominant non-scaling cost of the old row-split parallelism. -func gemmBand(r0, r1, jc, nc, pc, kc int, dst []float64, a Mat, n int, pa, pb []float64) { +func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pa, pb []T) { + mr0, nr0 := g.mr, g.nr + tile := pa[len(pa)-mr0*nr0:] // see paCap for ic := r0; ic < r1; ic += blockMC { // L1 row block of A mc := min(blockMC, r1-ic) - packA(a, ic, mc, pc, kc, pa) // pack A(mc x kc) -> MR-tall panels - for jr := 0; jr < nc; jr += NR { - nr := min(NR, nc-jr) - for ir := 0; ir < mc; ir += MR { - mr := min(MR, mc-ir) + g.packA(a, ic, mc, pc, kc, pa) // pack A(mc x kc) -> MR-tall panels + for jr := 0; jr < nc; jr += nr0 { + nr := min(nr0, nc-jr) + for ir := 0; ir < mc; ir += mr0 { + mr := min(mr0, mc-ir) c0 := (ic+ir)*n + jc + jr - if mr == MR && nr == NR { - gemmMicro(kc, pa[ir*kc:], pb[jr*kc:], dst[c0:], n) + if mr == mr0 && nr == nr0 { + g.micro(kc, pa[ir*kc:], pb[jr*kc:], dst[c0:], n) } else { - gemmEdge(kc, pa[ir*kc:], pb[jr*kc:], dst, c0, n, mr, nr) + g.gemmEdge(kc, pa[ir*kc:], pb[jr*kc:], dst, c0, n, mr, nr, tile) } } } @@ -162,7 +197,8 @@ func gemmBand(r0, r1, jc, nc, pc, kc int, dst []float64, a Mat, n int, pa, pb [] // columns of an edge panel zero-filled so the micro-kernel reads a full NR. // A unit column stride copies row runs; any other (a transposed B) walks each // column down its own unit-or-not row stride, writing the panel column-wise. -func packB(b Mat, pc, kc, jc, nc int, pb []float64) { +func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) { + NR := g.nr for jr := 0; jr < nc; jr += NR { nr := min(NR, nc-jr) dst := pb[jr*kc:] @@ -170,12 +206,8 @@ func packB(b Mat, pc, kc, jc, nc int, pb []float64) { for p := 0; p < kc; p++ { srcRow := b.Data[b.Off+(pc+p)*b.RS+jc+jr:] d := dst[p*NR:] - for c := 0; c < nr; c++ { - d[c] = srcRow[c] - } - for c := nr; c < NR; c++ { - d[c] = 0 - } + copy(d[:nr], srcRow[:nr]) + clear(d[nr:NR]) } continue } @@ -185,10 +217,9 @@ func packB(b Mat, pc, kc, jc, nc int, pb []float64) { dst[p*NR+c] = b.Data[src+p*b.RS] } } - for p := 0; p < kc; p++ { - d := dst[p*NR:] - for c := nr; c < NR; c++ { - d[c] = 0 + if nr < NR { + for p := 0; p < kc; p++ { + clear(dst[p*NR+nr : p*NR+NR]) } } } @@ -197,20 +228,19 @@ func packB(b Mat, pc, kc, jc, nc int, pb []float64) { // packA copies A[ic:ic+mc][pc:pc+kc] into MR-tall row panels: for each panel ir // the layout is pa[ir*kc + p*MR + r] = A(ic+ir+r, pc+p), with the r >= mr rows // of an edge panel zero-filled so the micro-kernel reads a full MR. -func packA(a Mat, ic, mc, pc, kc int, pa []float64) { +func (g *gemmKernel[T]) packA(a MatOf[T], ic, mc, pc, kc int, pa []T) { + MR := g.mr for ir := 0; ir < mc; ir += MR { mr := min(MR, mc-ir) dst := pa[ir*kc:] src := a.Off + (ic+ir)*a.RS + pc*a.CS for p := 0; p < kc; p++ { - d := dst[p*MR:] + d := dst[p*MR : p*MR+MR] at := src + p*a.CS for r := 0; r < mr; r++ { d[r] = a.Data[at+r*a.RS] } - for r := mr; r < MR; r++ { - d[r] = 0 - } + clear(d[mr:]) } } } @@ -222,9 +252,10 @@ func packA(a Mat, ic, mc, pc, kc int, pa []float64) { // by the same kernel, edge or not. The scalar loop this replaces ran ~20x slower // per element, and on amd64 (MR=6) every 256-row MC block ends in a 4-row edge // panel: it measured 40% of a 512^2 product there. -func gemmEdge(kc int, pa, pb, dst []float64, c0, ldc, mr, nr int) { - var tile [MR * NR]float64 - gemmMicro(kc, pa, pb, tile[:], NR) +func (g *gemmKernel[T]) gemmEdge(kc int, pa, pb, dst []T, c0, ldc, mr, nr int, tile []T) { + NR := g.nr + clear(tile) + g.micro(kc, pa, pb, tile, NR) for r := 0; r < mr; r++ { row := dst[c0+r*ldc : c0+r*ldc+nr] for c := range row { @@ -289,12 +320,28 @@ func MatMulP(dst, a, b []float64, m, k, n int) { // only read, through the packing, so a transposed or sliced operand costs no // copy beyond the one the packed GEMM makes anyway. func MatMulStridedP(dst []float64, a, b Mat, m, k, n int) { + gemm64.matMul(dst, a, b, m, k, n) +} + +// HasGemm32 reports whether this architecture and CPU have a float32 GEMM +// micro-kernel, which MatMul32StridedP requires. Without one, the caller +// multiplies in float64. +func HasGemm32() bool { return gemm32 != nil } + +// MatMul32StridedP is MatMulStridedP in float32, accumulating in float32 as a +// BLAS sgemm does. It requires HasGemm32. +func MatMul32StridedP(dst []float32, a, b Mat32, m, k, n int) { + gemm32.matMul(dst, a, b, m, k, n) +} + +func (g *gemmKernel[T]) matMul(dst []T, a, b MatOf[T], m, k, n int) { if m*n < GemmThreshold { - buf := getPackBuf() - packGemmRows(0, m, dst, a, b, k, n, buf) - putPackBuf(buf) + buf := g.getPackBuf() + g.packGemmRows(0, m, dst, a, b, k, n, buf) + g.putPackBuf(buf) return } + MR, NR := g.mr, g.nr w := runtime.GOMAXPROCS(0) if w > m { w = m @@ -336,9 +383,9 @@ func MatMulStridedP(dst []float64, a, b Mat, m, k, n int) { // pb is double-buffered, so block i can be packed while block i-1 is being // multiplied. Tiles of C are still accumulated in pc order, since block i's // bands wait for block i-1's. - pbs := [2][]float64{getPB(), getPB()} - defer putPB(pbs[0]) - defer putPB(pbs[1]) + pbs := [2][]T{g.getPB(), g.getPB()} + defer g.putPB(pbs[0]) + defer g.putPB(pbs[1]) type block struct{ jc, nc, pc, kc, chunks int } const chunk = 4 // B panels packed per claim @@ -358,8 +405,8 @@ func MatMulStridedP(dst []float64, a, b Mat, m, k, n int) { bandsDone := func(i int) bool { return i < 0 || prog[i].bandDone.Load() == int64(nBands) } work := func() { - pa := getPA() - defer putPA(pa) + pa := g.getPA() + defer g.putPA(pa) for i, bl := range blocks { // pbs[i%2] is free: its last reader was block i-2, and to get here // this worker passed block i-1's wait, which requires every band of @@ -371,7 +418,7 @@ func MatMulStridedP(dst []float64, a, b Mat, m, k, n int) { break } j0, j1 := c*chunk*NR, min((c+1)*chunk*NR, bl.nc) - packB(b, bl.pc, bl.kc, bl.jc+j0, j1-j0, pb[j0*bl.kc:]) + g.packB(b, bl.pc, bl.kc, bl.jc+j0, j1-j0, pb[j0*bl.kc:]) pr.packDone.Add(1) } spinUntil(func() bool { return pr.packDone.Load() == int64(bl.chunks) && bandsDone(i-1) }) @@ -381,13 +428,13 @@ func MatMulStridedP(dst []float64, a, b Mat, m, k, n int) { break } r0 := bi * bandRows - gemmBand(r0, min(r0+bandRows, m), bl.jc, bl.nc, bl.pc, bl.kc, dst, a, n, pa, pb) + g.gemmBand(r0, min(r0+bandRows, m), bl.jc, bl.nc, bl.pc, bl.kc, dst, a, n, pa, pb) pr.bandDone.Add(1) } } } var wg sync.WaitGroup - for g := 1; g < w; g++ { + for range w - 1 { wg.Add(1) go func() { defer wg.Done(); work() }() } @@ -400,9 +447,9 @@ func MatMulStridedP(dst []float64, a, b Mat, m, k, n int) { // the named serial entry point used by callers and tests that want the exact // non-parallel computation. func MatMul(dst, a, b []float64, m, k, n int) { - buf := getPackBuf() - packGemmRows(0, m, dst, rowMajor(a, k), rowMajor(b, n), k, n, buf) - putPackBuf(buf) + buf := gemm64.getPackBuf() + gemm64.packGemmRows(0, m, dst, rowMajor(a, k), rowMajor(b, n), k, n, buf) + gemm64.putPackBuf(buf) } // dotRange returns sum(a[i]*b[i]) over equal-length slices, using four diff --git a/internal/kernels/gemm32_amd64.go b/internal/kernels/gemm32_amd64.go new file mode 100644 index 0000000..e928375 --- /dev/null +++ b/internal/kernels/gemm32_amd64.go @@ -0,0 +1,24 @@ +package kernels + +// gemm32 is the float32 GEMM on the AVX2/FMA 6x16 kernel (gemmMicro6x16FMA in +// sum_amd64.s, generated by go-asmgen; see asmgen/amd64/gen.go). Without FMA +// there is none, and float32 products are computed in float64. +var gemm32 = newGemm32(useFMA) + +func newGemm32(fma bool) *gemmKernel[float32] { + if !fma { + return nil + } + return &gemmKernel[float32]{mr: 6, nr: 16, micro: gemmMicro32} +} + +// gemmMicro32 adds the 6 x 16 tile sum_p pa[p*6..]*pb[p*16..] into the C block +// at dst[0:], whose rows are ldc apart. pa, pb hold at least kc*6 and kc*16 +// packed (zero-padded) floats; dst has at least 5*ldc + 16 elements. +func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { + _, _, _ = pa[kc*6-1], pb[kc*16-1], dst[5*ldc+15] // the kernel's whole reach + gemmMicro6x16FMA(kc, &pa[0], &pb[0], &dst[0], ldc) +} + +//go:noescape +func gemmMicro6x16FMA(kc int, pa, pb, c *float32, ldc int) diff --git a/internal/kernels/gemm32_arm64.go b/internal/kernels/gemm32_arm64.go new file mode 100644 index 0000000..22f1a3d --- /dev/null +++ b/internal/kernels/gemm32_arm64.go @@ -0,0 +1,16 @@ +package kernels + +// gemm32 is the float32 GEMM on the NEON 8x8 kernel (gemmMicro8x8S in +// sum_arm64.s, generated by go-asmgen; see asmgen/arm64/gen.go). +var gemm32 = &gemmKernel[float32]{mr: 8, nr: 8, micro: gemmMicro32} + +// gemmMicro32 adds the 8 x 8 tile sum_p pa[p*8..]*pb[p*8..] into the C block at +// dst[0:], whose rows are ldc apart. pa, pb hold at least kc*8 packed +// (zero-padded) floats; dst has at least 7*ldc + 8 elements. +func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { + _, _, _ = pa[kc*8-1], pb[kc*8-1], dst[7*ldc+7] // the kernel's whole reach + gemmMicro8x8S(kc, &pa[0], &pb[0], &dst[0], ldc) +} + +//go:noescape +func gemmMicro8x8S(kc int, pa, pb, c *float32, ldc int) diff --git a/internal/kernels/gemm32_other.go b/internal/kernels/gemm32_other.go new file mode 100644 index 0000000..3c45cdf --- /dev/null +++ b/internal/kernels/gemm32_other.go @@ -0,0 +1,7 @@ +//go:build !amd64 && !arm64 + +package kernels + +// gemm32 is nil here: no float32 micro-kernel yet, so HasGemm32 is false and +// float32 products are computed in float64. +var gemm32 *gemmKernel[float32] diff --git a/internal/kernels/gemm32_test.go b/internal/kernels/gemm32_test.go new file mode 100644 index 0000000..15460b5 --- /dev/null +++ b/internal/kernels/gemm32_test.go @@ -0,0 +1,149 @@ +package kernels + +import ( + "math" + "math/rand" + "testing" +) + +// gemmOracle32 is gemmOracle in float32 arithmetic: the ikj order, each product +// and sum rounded to float32. +func gemmOracle32(dst []float32, a, b MatOf[float32], m, k, n int) { + for i := 0; i < m; i++ { + for p := 0; p < k; p++ { + av := a.Data[a.Off+i*a.RS+p*a.CS] + for j := 0; j < n; j++ { + dst[i*n+j] += av * b.Data[b.Off+p*b.RS+j*b.CS] + } + } + } +} + +func intMat32(m, k int, seed int64) []float32 { + r := rand.New(rand.NewSource(seed)) + a := make([]float32, m*k) + for i := range a { + a[i] = float32(r.Intn(17) - 8) + } + return a +} + +// assertGemm32 checks the float32 GEMM, serial and with the parallel band split +// forced, against the oracle bit for bit on integer-valued operands, whose +// products and sums are exact in float32 at these sizes. +func assertGemm32(t *testing.T, m, k, n int, a, b Mat32) { + t.Helper() + want := make([]float32, m*n) + gemmOracle32(want, a, b, m, k, n) + for _, gemm := range []int{1 << 30, 1} { + withThresholds(1<<20, gemm, func() { + got := make([]float32, m*n) + MatMul32StridedP(got, a, b, m, k, n) + for i := range want { + if got[i] != want[i] { + t.Fatalf("m=%d k=%d n=%d gemm=%d [%d]: %v != %v", m, k, n, gemm, i, got[i], want[i]) + } + } + }) + } +} + +// TestGemm32Correctness is TestGemmCorrectness for float32: every shape and +// block configuration that drives the edge tiles and the loop nest, on +// row-major operands and on a transposed, offset B view. +func TestGemm32Correctness(t *testing.T) { + g := gemm32 + if !HasGemm32() { + t.Skip("no float32 GEMM kernel here") + } + MR, NR := g.mr, g.nr + shapes := []struct{ m, k, n int }{ + {1, 1, 1}, {2, 3, 4}, {6, 4, 16}, {8, 8, 8}, {5, 7, 9}, {7, 7, 17}, + {9, 5, 11}, {16, 16, 16}, {17, 13, 19}, {3, 10, 1}, {1, 10, 3}, + {33, 33, 33}, {32, 64, 48}, + } + for _, s := range shapes { + a := rowMajor(intMat32(s.m, s.k, int64(s.m*100+s.k)), s.k) + b := rowMajor(intMat32(s.k, s.n, int64(s.n*7+s.k)), s.n) + // bt holds B transposed behind a 3-element offset; the view reads it + // back as B, through the packB column walk (CS != 1). + bt := Mat32{Data: make([]float32, 3+s.k*s.n), Off: 3, RS: 1, CS: s.k} + for p := 0; p < s.k; p++ { + for j := 0; j < s.n; j++ { + bt.Data[3+j*s.k+p] = b.Data[p*s.n+j] + } + } + for _, bv := range []Mat32{b, bt} { + assertGemm32(t, s.m, s.k, s.n, a, bv) + withBlocks(MR, 2, NR, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) + withBlocks(2*MR, 3, 2*NR, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) + withBlocks(MR+1, 2, NR+1, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) + withBlocks(3*MR-1, 5, 3*NR-1, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) + } + } + // Default blocks with MR not dividing blockMC, across the band split. + m, k, n := 300, 200, 260 + assertGemm32(t, m, k, n, rowMajor(intMat32(m, k, 1), k), rowMajor(intMat32(k, n, 2), n)) +} + +// TestGemm32Accuracy bounds the float32 GEMM on random data by the classical +// error bound of a float32 dot product, k*eps32*sum|a||b|, against the product +// computed in float64. +func TestGemm32Accuracy(t *testing.T) { + if gemm32 == nil { + t.Skip("no float32 GEMM kernel here") + } + r := rand.New(rand.NewSource(5)) + m, k, n := 70, 300, 50 + a, b := make([]float32, m*k), make([]float32, k*n) + for i := range a { + a[i] = float32(r.NormFloat64()) + } + for i := range b { + b[i] = float32(r.NormFloat64()) + } + got := make([]float32, m*n) + MatMul32StridedP(got, rowMajor(a, k), rowMajor(b, n), m, k, n) + for i := 0; i < m; i++ { + for j := 0; j < n; j++ { + var exact, abs float64 + for p := 0; p < k; p++ { + x := float64(a[i*k+p]) * float64(b[p*n+j]) + exact += x + abs += math.Abs(x) + } + if d := math.Abs(float64(got[i*n+j]) - exact); d > float64(k)*0x1p-24*abs { + t.Fatalf("[%d,%d]: %v vs %v (off %g, bound %g)", i, j, got[i*n+j], exact, d, float64(k)*0x1p-24*abs) + } + } + } +} + +// TestGemm32Specials: NaN and Inf in an operand reach exactly the outputs +// whose dot products they enter. +func TestGemm32Specials(t *testing.T) { + if gemm32 == nil { + t.Skip("no float32 GEMM kernel here") + } + m, k, n := 13, 9, 21 + for _, special := range []float32{float32(math.NaN()), float32(math.Inf(1))} { + a, b := intMat32(m, k, 3), intMat32(k, n, 4) + for i := range b { + if b[i] == 0 { + b[i] = 1 // keep Inf*0 out: the row of a meets every b entry + } + } + a[4*k+2] = special // row 4 + got := make([]float32, m*n) + MatMul32StridedP(got, rowMajor(a, k), rowMajor(b, n), m, k, n) + for i := 0; i < m; i++ { + for j := 0; j < n; j++ { + v := float64(got[i*n+j]) + bad := math.IsNaN(v) || math.IsInf(v, 0) + if bad != (i == 4) { + t.Fatalf("%v at row 4: [%d,%d] = %v", special, i, j, v) + } + } + } + } +} diff --git a/internal/kernels/gemm_amd64_test.go b/internal/kernels/gemm_amd64_test.go index 5b00f4e..67cfd66 100644 --- a/internal/kernels/gemm_amd64_test.go +++ b/internal/kernels/gemm_amd64_test.go @@ -15,3 +15,14 @@ func TestGemmSSE2Fallback(t *testing.T) { t.Run("float", TestGemmFloatTolerance) t.Run("dot", TestDotSIMD) } + +// TestGemm32NeedsFMA: without FMA there is no float32 kernel, and HasGemm32 +// sends float32 products to the float64 GEMM. +func TestGemm32NeedsFMA(t *testing.T) { + if newGemm32(false) != nil { + t.Fatal("a float32 GEMM without FMA") + } + if g := newGemm32(true); g == nil || g.mr != 6 || g.nr != 16 { + t.Fatalf("newGemm32(true) = %+v", g) + } +} diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index 1c55394..09a353d 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -535,6 +535,148 @@ TEXT ·gemmMicro6x8FMA(SB), NOSPLIT, $0-40 VZEROUPPER RET +TEXT ·gemmMicro6x16FMA(SB), NOSPLIT, $0-40 + MOVQ kc+0(FP), CX + MOVQ pa+8(FP), SI + MOVQ pb+16(FP), DI + MOVQ c+24(FP), DX + MOVQ ldc+32(FP), R8 + SHLQ $2, R8 + MOVQ DX, R9 + PREFETCHT0 (R9) + PREFETCHT0 63(R9) + ADDQ R8, R9 + PREFETCHT0 (R9) + PREFETCHT0 63(R9) + ADDQ R8, R9 + PREFETCHT0 (R9) + PREFETCHT0 63(R9) + ADDQ R8, R9 + PREFETCHT0 (R9) + PREFETCHT0 63(R9) + ADDQ R8, R9 + PREFETCHT0 (R9) + PREFETCHT0 63(R9) + ADDQ R8, R9 + PREFETCHT0 (R9) + PREFETCHT0 63(R9) + VXORPS Y0, Y0, Y0 + VXORPS Y1, Y1, Y1 + VXORPS Y2, Y2, Y2 + VXORPS Y3, Y3, Y3 + VXORPS Y4, Y4, Y4 + VXORPS Y5, Y5, Y5 + VXORPS Y6, Y6, Y6 + VXORPS Y7, Y7, Y7 + VXORPS Y8, Y8, Y8 + VXORPS Y9, Y9, Y9 + VXORPS Y10, Y10, Y10 + VXORPS Y11, Y11, Y11 + MOVQ CX, R10 + SHRQ $1, R10 + JZ stail + skloop: + PREFETCHT0 384(SI) + VMOVUPS 0(DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS 0(SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS 4(SI), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + VBROADCASTSS 8(SI), Y14 + VFMADD231PS Y12, Y14, Y4 + VFMADD231PS Y13, Y14, Y5 + VBROADCASTSS 12(SI), Y14 + VFMADD231PS Y12, Y14, Y6 + VFMADD231PS Y13, Y14, Y7 + VBROADCASTSS 16(SI), Y14 + VFMADD231PS Y12, Y14, Y8 + VFMADD231PS Y13, Y14, Y9 + VBROADCASTSS 20(SI), Y14 + VFMADD231PS Y12, Y14, Y10 + VFMADD231PS Y13, Y14, Y11 + VMOVUPS 64(DI), Y12 + VMOVUPS 96(DI), Y13 + VBROADCASTSS 24(SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS 28(SI), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + VBROADCASTSS 32(SI), Y14 + VFMADD231PS Y12, Y14, Y4 + VFMADD231PS Y13, Y14, Y5 + VBROADCASTSS 36(SI), Y14 + VFMADD231PS Y12, Y14, Y6 + VFMADD231PS Y13, Y14, Y7 + VBROADCASTSS 40(SI), Y14 + VFMADD231PS Y12, Y14, Y8 + VFMADD231PS Y13, Y14, Y9 + VBROADCASTSS 44(SI), Y14 + VFMADD231PS Y12, Y14, Y10 + VFMADD231PS Y13, Y14, Y11 + ADDQ $48, SI + ADDQ $128, DI + DECQ R10 + JNZ skloop + stail: + TESTQ $1, CX + JZ sstore + VMOVUPS 0(DI), Y12 + VMOVUPS 32(DI), Y13 + VBROADCASTSS 0(SI), Y14 + VFMADD231PS Y12, Y14, Y0 + VFMADD231PS Y13, Y14, Y1 + VBROADCASTSS 4(SI), Y14 + VFMADD231PS Y12, Y14, Y2 + VFMADD231PS Y13, Y14, Y3 + VBROADCASTSS 8(SI), Y14 + VFMADD231PS Y12, Y14, Y4 + VFMADD231PS Y13, Y14, Y5 + VBROADCASTSS 12(SI), Y14 + VFMADD231PS Y12, Y14, Y6 + VFMADD231PS Y13, Y14, Y7 + VBROADCASTSS 16(SI), Y14 + VFMADD231PS Y12, Y14, Y8 + VFMADD231PS Y13, Y14, Y9 + VBROADCASTSS 20(SI), Y14 + VFMADD231PS Y12, Y14, Y10 + VFMADD231PS Y13, Y14, Y11 + sstore: + VADDPS (DX), Y0, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y1, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R8, DX + VADDPS (DX), Y2, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y3, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R8, DX + VADDPS (DX), Y4, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y5, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R8, DX + VADDPS (DX), Y6, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y7, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R8, DX + VADDPS (DX), Y8, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y9, Y15 + VMOVUPS Y15, 32(DX) + ADDQ R8, DX + VADDPS (DX), Y10, Y15 + VMOVUPS Y15, (DX) + VADDPS 32(DX), Y11, Y15 + VMOVUPS Y15, 32(DX) + VZEROUPPER + RET + TEXT ·dotFMA(SB), NOSPLIT, $0-32 MOVQ a+0(FP), SI MOVQ b+8(FP), DI diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index fc8bb74..b938caf 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -600,6 +600,93 @@ TEXT ·gemmMicro4x8(SB), NOSPLIT, $0-40 VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5) RET +TEXT ·gemmMicro8x8S(SB), NOSPLIT, $0-40 + MOVD kc+0(FP), R0 + MOVD pa+8(FP), R1 + MOVD pb+16(FP), R2 + MOVD c+24(FP), R3 + MOVD ldc+32(FP), R4 + LSL $2, R4, R4 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + VEOR V8.B16, V8.B16, V8.B16 + VEOR V9.B16, V9.B16, V9.B16 + VEOR V10.B16, V10.B16, V10.B16 + VEOR V11.B16, V11.B16, V11.B16 + VEOR V12.B16, V12.B16, V12.B16 + VEOR V13.B16, V13.B16, V13.B16 + VEOR V14.B16, V14.B16, V14.B16 + VEOR V15.B16, V15.B16, V15.B16 + CBZ R0, sstore + skloop: + VLD1.P 32(R2), [V16.S4, V17.S4] + VLD1.P 32(R1), [V18.S4, V19.S4] + WORD $0x4F921200 // FMLA V0.4S, V16.4S, V18.S[0] + WORD $0x4F921221 // FMLA V1.4S, V17.4S, V18.S[0] + WORD $0x4FB21202 // FMLA V2.4S, V16.4S, V18.S[1] + WORD $0x4FB21223 // FMLA V3.4S, V17.4S, V18.S[1] + WORD $0x4F921A04 // FMLA V4.4S, V16.4S, V18.S[2] + WORD $0x4F921A25 // FMLA V5.4S, V17.4S, V18.S[2] + WORD $0x4FB21A06 // FMLA V6.4S, V16.4S, V18.S[3] + WORD $0x4FB21A27 // FMLA V7.4S, V17.4S, V18.S[3] + WORD $0x4F931208 // FMLA V8.4S, V16.4S, V19.S[0] + WORD $0x4F931229 // FMLA V9.4S, V17.4S, V19.S[0] + WORD $0x4FB3120A // FMLA V10.4S, V16.4S, V19.S[1] + WORD $0x4FB3122B // FMLA V11.4S, V17.4S, V19.S[1] + WORD $0x4F931A0C // FMLA V12.4S, V16.4S, V19.S[2] + WORD $0x4F931A2D // FMLA V13.4S, V17.4S, V19.S[2] + WORD $0x4FB31A0E // FMLA V14.4S, V16.4S, V19.S[3] + WORD $0x4FB31A2F // FMLA V15.4S, V17.4S, V19.S[3] + SUB $1, R0 + CBNZ R0, skloop + sstore: + VLD1 (R3), [V16.S4, V17.S4] + VFADD V0.S4, V16.S4, V16.S4 + VFADD V1.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V2.S4, V16.S4, V16.S4 + VFADD V3.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V4.S4, V16.S4, V16.S4 + VFADD V5.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V6.S4, V16.S4, V16.S4 + VFADD V7.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V8.S4, V16.S4, V16.S4 + VFADD V9.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V10.S4, V16.S4, V16.S4 + VFADD V11.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V12.S4, V16.S4, V16.S4 + VFADD V13.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + ADD R4, R3, R3 + VLD1 (R3), [V16.S4, V17.S4] + VFADD V14.S4, V16.S4, V16.S4 + VFADD V15.S4, V17.S4, V17.S4 + VST1 [V16.S4, V17.S4], (R3) + RET + TEXT ·blocks8NEON(SB), NOSPLIT, $0-24 MOVD r+0(FP), R0 MOVD a+8(FP), R1 diff --git a/linalg.go b/linalg.go index 13cee0d..261e254 100644 --- a/linalg.go +++ b/linalg.go @@ -51,17 +51,23 @@ func (a *Array) MatMul(b *Array) (*Array, error) { } // matmulTyped is MatMul for operands of other dtypes, in their common dtype. -// Float32 runs on the blocked SIMD float64 GEMM: the operands are widened -// (exact, O(n²)), multiplied and accumulated in float64, and each element is -// rounded once to float32. That is at least as accurate as accumulating in -// float32, as a BLAS sgemm does, and on a POWER9 it was 9x faster at 256² than -// the triple loop below, which the other dtypes keep. +// +// Float32 runs on the packed float32 GEMM where there is a float32 kernel +// (arm64, and amd64 with FMA): it accumulates in float32, as NumPy's BLAS +// sgemm does, at twice the SIMD lanes of float64. Elsewhere it runs on the +// float64 GEMM: the operands are widened (exact, O(n²)), multiplied and +// accumulated in float64, and each element is rounded once to float32, which +// is at least as accurate; on a POWER9 that was 9x faster at 256² than the +// triple loop below, which the other dtypes keep. func matmulTyped(a, b *Array) *Array { dt := ResultType(a.dtype, b.dtype) switch dt { case Float64: return a.toFloat64().matmulFloat64(b.toFloat64()) case Float32: + if useGemm32 { + return matmulFloat32(a, b) + } return a.toFloat64().matmulFloat64(b.toFloat64()).AsType(Float32) } m, k, n := a.shape[0], a.shape[1], b.shape[1] @@ -70,6 +76,28 @@ func matmulTyped(a, b *Array) *Array { return a.result(d, []int{m, n}) } +// useGemm32 selects the float32 GEMM. It is a var so tests can force the +// widening path on hardware that has the kernel (never the reverse). +var useGemm32 = kernels.HasGemm32() + +// matmulFloat32 multiplies two validated 2-D operands whose common dtype is +// Float32 on the float32 GEMM. A Float32 operand is read in place through its +// strides, like matView; another dtype is converted first. +func matmulFloat32(a, b *Array) *Array { + m, k, n := a.shape[0], a.shape[1], b.shape[1] + dst := make([]float32, m*n) // MatMul32StridedP accumulates into dst + kernels.MatMul32StridedP(dst, a.matView32(), b.matView32(), m, k, n) + return a.result(dst, []int{m, n}) +} + +// matView32 is matView for an operand of the float32 GEMM. +func (a *Array) matView32() kernels.Mat32 { + if a.dtype != Float32 { + return kernels.Mat32{Data: a.operandStore(Float32, a.shape).([]float32), RS: a.shape[1], CS: 1} + } + return kernels.Mat32{Data: typed[float32](a), Off: a.offset, RS: a.strides[0], CS: a.strides[1]} +} + // matmulFloat64 is matmul2D on two validated 2-D Float64 arrays. func (a *Array) matmulFloat64(b *Array) *Array { return matmul2D(a, b, a.shape[0], a.shape[1], b.shape[1]) diff --git a/linalg_test.go b/linalg_test.go index eb4c42f..6232165 100644 --- a/linalg_test.go +++ b/linalg_test.go @@ -64,6 +64,43 @@ func TestMatMulFloat32(t *testing.T) { } } +// TestMatMulFloat32Operands multiplies float32 operands that the float32 GEMM +// reads in place (a transposed view) or converts first (an Int8 operand, whose +// common dtype with Float32 is Float32), on integer data where every path is +// exact. +func TestMatMulFloat32Operands(t *testing.T) { + m, k, n := 9, 13, 17 + x := make([]float32, k*m) // holds aᵀ + y := make([]float32, k*n) + yi := make([]int8, k*n) + for i := range x { + x[i] = float32(i%11 - 5) + } + for i := range y { + yi[i] = int8(i%7 - 3) + y[i] = float32(yi[i]) + } + at := mustArr(t, ok(FromSlice(x, k, m))).Transpose() + want := make([]float32, m*n) + matmulNum(want, data[float32](t, at), y, m, k, n) + saved := useGemm32 + defer func() { useGemm32 = saved }() + for _, native := range []bool{saved, false} { // the float32 GEMM, then widening + useGemm32 = native + for _, b := range []*Array{mustArr(t, ok(FromSlice(y, k, n))), mustArr(t, ok(FromSlice(yi, k, n)))} { + r := mustArr(t, ok(at.MatMul(b))) + if r.DType() != Float32 { + t.Fatalf("%v operand: dtype %v, want float32", b.DType(), r.DType()) + } + for i, g := range data[float32](t, r) { + if g != want[i] { + t.Fatalf("native=%v %v operand [%d] = %v, want %v", native, b.DType(), i, g, want[i]) + } + } + } + } +} + func TestMatMulTransposedView(t *testing.T) { // A.T @ A where A is (3x2): A.T is a (2x3) strided view, exercising // materialize inside matmul2D. Result (2x2) == [[35,44],[44,56]] (numpy). From 59b9df3fa45cf006a01cc72e8faceebfa16dbb68 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 26 Oct 2026 21:31:28 +0100 Subject: [PATCH 2/2] perf(gemm): pack a row-major A row by row; docs for the float32 GEMM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packA gathered A column by column, paying its loop and bounds checks every MR elements. At 64² that took 33% of a float32 product, more than the micro-kernel. A row-major A (CS == 1) is now read one contiguous row at a time and scattered into its lane of the panel. packB goes back to element loops: copy and clear on NR-element runs are runtime calls. abbench, 10 rounds, one core, against main: Zen 3 float32 -61/-56/-55% (64²/256²/1024²), float64 -5.8/-4.2/~ N1 float32 -64/-60/-57%, float64 ~/~/~ perf.md gains the measurements and the NumPy 2.5.4 comparison (still 1.1-1.9x behind OpenBLAS: an open defect). README, plan and SECURITY are updated too. Co-Authored-By: Claude Opus 5.5 --- README.md | 9 ++++--- SECURITY.md | 12 +++++++++ docs/perf.md | 55 ++++++++++++++++++++++++++++++++++++++-- docs/plan-ndarray.md | 6 +++-- internal/kernels/gemm.go | 42 +++++++++++++++++++++++++----- 5 files changed, 109 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 0100c59..e8c1bb0 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every 64-bit target, and `Max`/`Min` on every 64-bit target, bit-identical to its - scalar loops; its `MatMul` runs on the float64 GEMM (accumulated in float64, - rounded once) + scalar loops. Its `MatMul` runs on a native float32 GEMM on arm64 (NEON) + and amd64 with FMA (AVX2), accumulating in float32 like NumPy's sgemm, and + elsewhere on the float64 GEMM (accumulated in float64, rounded once) ([docs/design-dtypes.md](docs/design-dtypes.md)). - **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/ `Eye`/`Identity`. @@ -154,8 +155,8 @@ and `Max` by 92–96% ([docs/perf.md](docs/perf.md), *SIMD coverage*). It is a > pair Go supports; v0.1.0 and v0.2.0 were also run on real amd64, arm64, > ppc64le, riscv64 and loong64 hardware, v0.3.0 on real amd64, arm64, ppc64le > and loong64. See **[docs/plan-ndarray.md](docs/plan-ndarray.md)** -> for the roadmap (float32 SIMD for max/min and the products; linear algebra -> decompositions). +> for the roadmap (a native float32 GEMM on the other four 64-bit targets; +> linear algebra decompositions). ## Why this module? diff --git a/SECURITY.md b/SECURITY.md index 8386786..4eaa8e1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,18 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-26 (v0.26.0).** The float32 GEMM kernels, `gemmMicro8x8S` (arm64) +and `gemmMicro6x16FMA` (amd64, behind the FMA probe), read `kc·MR` and `kc·NR` +packed floats and read and write one MR×NR tile of C. Their Go wrapper +bounds-checks all three reaches before the call. They are fenced (PROT_NONE +on both sides of each operand) and tested against a float32 oracle. A +sabotaged kernel failed every CI job that runs it. The edge tile now lives +in the worker's pooled A buffer: its capacity adds MR·NR elements, and +`gemmBand` slices the tile from the end. The operand views are the same +`Off/RS/CS` triples the float64 GEMM reads, built from the array's own +strides. A non-Float32 operand is converted to a fresh contiguous copy +first. Shapes are validated (overflow included) before any allocation. + **2026-10-26 (v0.25.0).** The loong64 extremum kernels (`maxLSX`, `minLSX`, `max32LSX`, `min32LSX`) read the whole blocks of `a` behind the `useLASX` gate. They write one 16-byte vector to an `out` array on the wrapper's stack, diff --git a/docs/perf.md b/docs/perf.md index aafa13c..d637414 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1189,6 +1189,55 @@ float32 is now within the widening of float64: 86.7 against 54.0 µs at 64², lanes per register and half the bytes, could go further; it is the next step for products. +### A native float32 GEMM on amd64 and arm64 (2026-10-26) + +The packed GEMM driver (packing, cache blocking, the parallel bands) is now +generic over float32 and float64. Each element type brings its micro-kernel, +its tile and its pack-buffer pools. float32 gets two kernels, generated by +go-asmgen: + +- **arm64:** `gemmMicro8x8S`, a NEON 8×8 tile. It has 16 accumulators and + uses the by-element `FMLA Vd.4S, Vn.4S, Vm.S[i]`, encoded as a `WORD` and + checked against `go tool objdump`. +- **amd64:** `gemmMicro6x16FMA`, the BLIS Haswell sgemm shape: a 6×16 tile, + `VBROADCASTSS` and `VFMADD231PS` into 12 YMM accumulators. It runs only + where the CPU has FMA. + +float32 `MatMul` (and `Dot`, which goes through it) now accumulates in +float32, as NumPy's sgemm does. Without a kernel (no FMA, or the other four +targets) it still widens to the float64 GEMM. The tests check it against a +float32 ikj oracle, bit for bit on integer data, in every block +configuration, on a transposed and offset B view. On random data they bound +it by k·eps32·Σ|ab|. A sabotaged kernel (a wrong A lane on arm64, a wrong A +row on amd64) failed every job that runs one. + +Profiling the first version at 64² found `packA` at 33% of the time, ahead +of the micro-kernel. It gathered A column by column and paid its loop and +bounds checks every MR elements. A row-major A is now read one contiguous +row at a time, which also speeds up float64. + +| `MatMul` (`abbench`, 10 rounds, 1 core) | 64² | 256² | 1024² | +|---|---:|---:|---:| +| Zen 3, float32 | **−61%** | **−56%** | **−55%** | +| Zen 3, float64 | −5.8% | −4.2% | ~ | +| Neoverse N1, float32 | **−64%** | **−60%** | **−57%** | +| Neoverse N1, float64 | ~ | ~ | ~ | + +Against NumPy 2.5.4 (OpenBLAS, `OPENBLAS_NUM_THREADS=1`, the same core): + +| | 64² | 256² | 1024² | +|---|---:|---:|---:| +| Zen 3 float32, ours / NumPy | 15.0 / 8.1 µs | 485 / 366 µs | 26.0 / 23.1 ms | +| Zen 3 float64 | 23.2 / 15.8 µs | 866 / 795 µs | 51.0 / 50.0 ms | +| N1 float32 | 30.4 / 16.2 µs | 1.19 / 0.94 ms | 68.2 / 60.6 ms | +| N1 float64 | 48.2 / 31.3 µs | 2.46 / 1.95 ms | 154 / 123 ms | + +**Open defect: products are still slower than OpenBLAS on one core**, by +1.1–1.3× at 1024² and up to 1.9× at 64². The gap is widest where the work +around the kernel weighs most: packing, edge tiles, and the result's +allocation and zeroing. The kernels for ppc64le, s390x, riscv64 and loong64 +are still to write. + **Open: the allocating operations.** `Add` is still 1.5–3.2× slower than NumPy's `a+b`, for float64 as well. The kernel is no longer the difference, as `AddInto` shows; the allocation is. It splits like this, measured @@ -1224,12 +1273,14 @@ What does remove it: `VFMADD231PD`, C and A prefetched) when go-asmgen's CPUID probe reports FMA with the OS saving YMM state, else a pair of SSE2 6×4 `MULPD`+`ADDPD` tiles over the same packing. Generated by go-asmgen and validated per-arch in CI; - the FMA kernel on real Cascade Lake and Zen 3 hosts. + the FMA kernel on real Cascade Lake and Zen 3 hosts. float32 products have + their own FMA kernel, `gemmMicro6x16FMA` (6×16, `VFMADD231PS`). - **arm64 (NEON)** ships hand-vectorized `sum`, `dot`, **packed `sqrt`** (`VFSQRT`), the **elementwise `add`/`sub`/`mul`/`div`** (`VFADD`/`VFSUB`/ `VFMUL`/`VFDIV`), **`max`/`min`** (`VFMAX`/`VFMIN`, four D2 accumulators from 8 elements up) and the **GEMM micro-kernel** (`gemmMicro4x8`: a 4×8 tile, 16 D2 - accumulators, using the **by-lane FMLA** `FMLA Vd.2D, Vn.2D, Vm.D[i]`). Only + accumulators, using the **by-lane FMLA** `FMLA Vd.2D, Vn.2D, Vm.D[i]`), with + a float32 one, `gemmMicro8x8S` (8×8, `FMLA Vd.4S, Vn.4S, Vm.S[i]`). Only the indexed-element FMLA is still a raw `WORD` (16 of them): Go's assembler does not name it yet (pending as CL 764224). diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index 9337eb8..f422d71 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -159,8 +159,10 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). `Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go for float32: nothing in `Max`/`Min` any more (v0.20.0 to v0.25.0 vectorised them on all six 64-bit targets, for float32 and float64), and the transcendental ufuncs. `MatMul` runs on the float64 - GEMM since v0.21.0, 77–90% faster than the triple loop; a native float32 - GEMM is still to write. + GEMM since v0.21.0, 77–90% faster than the triple loop, and since v0.26.0 + on a native float32 GEMM on arm64 (NEON 8×8) and amd64 with FMA (6×16), + a further 53–61% faster. The float32 kernels for ppc64le, s390x, riscv64 + and loong64 are still to write. - **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic, comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`, `Where` and the per-axis reductions, in every dtype. Not done: a public diff --git a/internal/kernels/gemm.go b/internal/kernels/gemm.go index 7326284..c3142d8 100644 --- a/internal/kernels/gemm.go +++ b/internal/kernels/gemm.go @@ -203,11 +203,17 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) { nr := min(NR, nc-jr) dst := pb[jr*kc:] if b.CS == 1 { + // Element loops, not copy and clear: on runs this short (NR + // elements) each of those is a runtime call. for p := 0; p < kc; p++ { srcRow := b.Data[b.Off+(pc+p)*b.RS+jc+jr:] d := dst[p*NR:] - copy(d[:nr], srcRow[:nr]) - clear(d[nr:NR]) + for c := 0; c < nr; c++ { + d[c] = srcRow[c] + } + for c := nr; c < NR; c++ { + d[c] = 0 + } } continue } @@ -217,9 +223,10 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) { dst[p*NR+c] = b.Data[src+p*b.RS] } } - if nr < NR { - for p := 0; p < kc; p++ { - clear(dst[p*NR+nr : p*NR+NR]) + for p := 0; p < kc; p++ { + d := dst[p*NR:] + for c := nr; c < NR; c++ { + d[c] = 0 } } } @@ -234,13 +241,34 @@ func (g *gemmKernel[T]) packA(a MatOf[T], ic, mc, pc, kc int, pa []T) { mr := min(MR, mc-ir) dst := pa[ir*kc:] src := a.Off + (ic+ir)*a.RS + pc*a.CS + if a.CS == 1 { + // Row-major A: read each row as one contiguous run and scatter it + // into its lane of the panel. The column-wise walk below spends + // its loop and bounds checks on every MR elements; at 64² it + // took longer than the micro-kernel. + for r := 0; r < mr; r++ { + row := a.Data[src+r*a.RS : src+r*a.RS+kc] + d := dst[r : r+(kc-1)*MR+1] + for p, v := range row { + d[p*MR] = v + } + } + for r := mr; r < MR; r++ { + for p := 0; p < kc; p++ { + dst[p*MR+r] = 0 + } + } + continue + } for p := 0; p < kc; p++ { - d := dst[p*MR : p*MR+MR] + d := dst[p*MR:] at := src + p*a.CS for r := 0; r < mr; r++ { d[r] = a.Data[at+r*a.RS] } - clear(d[mr:]) + for r := mr; r < MR; r++ { + d[r] = 0 + } } } }