From d63170bea8ce626ca7ed6f854a56eb1753da0beb Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 27 Oct 2026 21:32:35 +0100 Subject: [PATCH 1/2] perf(gemm): native float32 GEMM on s390x (z14 VXE, 4x16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gemmMicro4x16VXE is the float64 4x8 kernel at twice the lanes: sixteen four-lane accumulators, the B row in one VLM, each A value replicated with VLREPF, and VFMASB, which go-asmgen encodes because Go's assembler has no short-format vector float op. It needs the vector-enhancements facility 1, so it is gated on useVXE like the other float32 kernels. Without it, float32 products still widen to the float64 GEMM. IBM Z (LinuxONE, VXE), abbench 8 rounds, one core, against v0.28.0: -58/-54/-50% at 64²/256²/1024². Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/s390x/gen.go | 42 +++++++++++++++ internal/kernels/gemm32_other.go | 2 +- internal/kernels/gemm32_s390x.go | 25 +++++++++ internal/kernels/sum_s390x.s | 79 ++++++++++++++++++++++++++++ internal/kernels/vx_s390x_test.go | 11 ++++ 5 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 internal/kernels/gemm32_s390x.go diff --git a/internal/kernels/asmgen/s390x/gen.go b/internal/kernels/asmgen/s390x/gen.go index fc55c1a..b3d4d46 100644 --- a/internal/kernels/asmgen/s390x/gen.go +++ b/internal/kernels/asmgen/s390x/gen.go @@ -38,6 +38,7 @@ func main() { f.Add(bin32Kernel(k.name[:3]+"32VXE", k.name[:3], k.sca+"S")) } f.Add(gemmKernel()) + f.Add(gemm32Kernel()) f.Add(extremumKernel("maxVXE", "VFMAXDB", "WFMAXDB")) f.Add(extremumKernel("minVXE", "VFMINDB", "WFMINDB")) f.Add(blocks8Kernel()) @@ -285,6 +286,47 @@ func gemmKernel() *emit.Function { return b.Ret().Func() } +// gemm32Kernel builds gemmMicro4x16VXE(kc int, pa, pb, c *float32, ldc int) +// for kc >= 1: C[r][0:16] += sum_p pa[p*4+r] * pb[p*16:p*16+16] for r < 4, the +// rows of C ldc float32s apart. gemmKernel at twice the lanes: sixteen +// four-lane accumulators V0..V15 (row r in V4r..V4r+3), the B row in one VLM, +// each A value replicated across a register with VLREPF, and VFMASB, the +// vector-enhancements facility 1 (z14) fused multiply-add, which go-asmgen +// encodes because Go's assembler has no short-format vector float op. +func gemm32Kernel() *emit.Function { + sig := s390x.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, + []s390x.Type{s390x.Int64, s390x.Ptr, s390x.Ptr, s390x.Ptr, s390x.Int64}, nil, nil) + b := s390x.NewFunc("gemmMicro4x16VXE", sig, 0) + b.LoadArg("kc", "R5").LoadArg("pa", "R2").LoadArg("pb", "R3"). + LoadArg("c", "R1").LoadArg("ldc", "R4") + for r := 0; r < 16; r++ { + b.Raw("VZERO V%d", r) + } + b.Raw("loop:").Raw("VLM (R3), V16, V19") // the B row: 16 floats + for r := 0; r < 4; r++ { + b.Raw("VLREPF %d(R2), V%d", 4*r, 20+r) // a_r in all four lanes + } + for r := 0; r < 4; r++ { + for j := 0; j < 4; j++ { + b.VFMASB(4*r+j, 20+r, 16+j, 4*r+j) // acc += a_r * b + } + } + b.Raw("ADD $16, R2").Raw("ADD $64, R3").Raw("SUB $1, R5"). + Raw("CMP R5, $0").Raw("BNE loop"). + Raw("SLD $2, R4") // row stride in bytes + for r := 0; r < 4; r++ { + b.Raw("VLM (R1), V24, V27") + for j := 0; j < 4; j++ { + b.VFASB(24+j, 24+j, 4*r+j) // C row += acc_r + } + b.Raw("VSTM V24, V27, (R1)") + if r < 3 { + b.Raw("ADD R4, R1") + } + } + return b.Ret().Func() +} + // extremumKernel builds NAME(a *float64, n int) float64, the maximum (or // minimum) of a, for n >= 8. VFMAXDB/VFMINDB and their scalar WF forms come // with the vector-enhancements facility 1 (z14), so the Go side gates these diff --git a/internal/kernels/gemm32_other.go b/internal/kernels/gemm32_other.go index bfdd10e..22d2440 100644 --- a/internal/kernels/gemm32_other.go +++ b/internal/kernels/gemm32_other.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 && !ppc64le +//go:build !amd64 && !arm64 && !ppc64le && !s390x package kernels diff --git a/internal/kernels/gemm32_s390x.go b/internal/kernels/gemm32_s390x.go new file mode 100644 index 0000000..9677a5f --- /dev/null +++ b/internal/kernels/gemm32_s390x.go @@ -0,0 +1,25 @@ +package kernels + +// gemm32 is the float32 GEMM on the 4x16 vector kernel (gemmMicro4x16VXE in +// sum_s390x.s, generated by go-asmgen; see asmgen/s390x/gen.go). Its VFMASB is +// the vector-enhancements facility 1 (z14): without it there is none, and +// float32 products are computed in float64. +var gemm32 = newGemm32(useVXE) + +func newGemm32(vxe bool) *gemmKernel[float32] { + if !vxe { + return nil + } + return &gemmKernel[float32]{mr: 4, nr: 16, micro: gemmMicro32} +} + +// gemmMicro32 adds the 4 x 16 tile sum_p pa[p*4..]*pb[p*16..] into the C block +// at dst[0:], whose rows are ldc apart. pa, pb hold at least kc*4 and kc*16 +// packed (zero-padded) floats; dst has at least 3*ldc + 16 elements. +func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { + _, _, _ = pa[kc*4-1], pb[kc*16-1], dst[3*ldc+15] // the kernel's whole reach + gemmMicro4x16VXE(kc, &pa[0], &pb[0], &dst[0], ldc) +} + +//go:noescape +func gemmMicro4x16VXE(kc int, pa, pb, c *float32, ldc int) diff --git a/internal/kernels/sum_s390x.s b/internal/kernels/sum_s390x.s index c991894..46619e0 100644 --- a/internal/kernels/sum_s390x.s +++ b/internal/kernels/sum_s390x.s @@ -471,6 +471,85 @@ TEXT ·gemmMicro4x8VX(SB), NOSPLIT, $0-40 VSTM V24, V27, (R1) RET +TEXT ·gemmMicro4x16VXE(SB), NOSPLIT, $0-40 + MOVD kc+0(FP), R5 + MOVD pa+8(FP), R2 + MOVD pb+16(FP), R3 + MOVD c+24(FP), R1 + MOVD ldc+32(FP), R4 + VZERO V0 + VZERO V1 + VZERO V2 + VZERO V3 + VZERO V4 + VZERO V5 + VZERO V6 + VZERO V7 + VZERO V8 + VZERO V9 + VZERO V10 + VZERO V11 + VZERO V12 + VZERO V13 + VZERO V14 + VZERO V15 + loop: + VLM (R3), V16, V19 + VLREPF 0(R2), V20 + VLREPF 4(R2), V21 + VLREPF 8(R2), V22 + VLREPF 12(R2), V23 + WORD $0xe7040200; BYTE $0x06; BYTE $0x8f // VFMASB V20, V16, V0, V0 + WORD $0xe7141200; BYTE $0x16; BYTE $0x8f // VFMASB V20, V17, V1, V1 + WORD $0xe7242200; BYTE $0x26; BYTE $0x8f // VFMASB V20, V18, V2, V2 + WORD $0xe7343200; BYTE $0x36; BYTE $0x8f // VFMASB V20, V19, V3, V3 + WORD $0xe7450200; BYTE $0x46; BYTE $0x8f // VFMASB V21, V16, V4, V4 + WORD $0xe7551200; BYTE $0x56; BYTE $0x8f // VFMASB V21, V17, V5, V5 + WORD $0xe7652200; BYTE $0x66; BYTE $0x8f // VFMASB V21, V18, V6, V6 + WORD $0xe7753200; BYTE $0x76; BYTE $0x8f // VFMASB V21, V19, V7, V7 + WORD $0xe7860200; BYTE $0x86; BYTE $0x8f // VFMASB V22, V16, V8, V8 + WORD $0xe7961200; BYTE $0x96; BYTE $0x8f // VFMASB V22, V17, V9, V9 + WORD $0xe7a62200; BYTE $0xa6; BYTE $0x8f // VFMASB V22, V18, V10, V10 + WORD $0xe7b63200; BYTE $0xb6; BYTE $0x8f // VFMASB V22, V19, V11, V11 + WORD $0xe7c70200; BYTE $0xc6; BYTE $0x8f // VFMASB V23, V16, V12, V12 + WORD $0xe7d71200; BYTE $0xd6; BYTE $0x8f // VFMASB V23, V17, V13, V13 + WORD $0xe7e72200; BYTE $0xe6; BYTE $0x8f // VFMASB V23, V18, V14, V14 + WORD $0xe7f73200; BYTE $0xf6; BYTE $0x8f // VFMASB V23, V19, V15, V15 + ADD $16, R2 + ADD $64, R3 + SUB $1, R5 + CMP R5, $0 + BNE loop + SLD $2, R4 + VLM (R1), V24, V27 + WORD $0xe7880000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V0, V24 + WORD $0xe7991000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V1, V25 + WORD $0xe7aa2000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V2, V26 + WORD $0xe7bb3000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V3, V27 + VSTM V24, V27, (R1) + ADD R4, R1 + VLM (R1), V24, V27 + WORD $0xe7884000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V4, V24 + WORD $0xe7995000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V5, V25 + WORD $0xe7aa6000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V6, V26 + WORD $0xe7bb7000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V7, V27 + VSTM V24, V27, (R1) + ADD R4, R1 + VLM (R1), V24, V27 + WORD $0xe7888000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V8, V24 + WORD $0xe7999000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V9, V25 + WORD $0xe7aaa000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V10, V26 + WORD $0xe7bbb000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V11, V27 + VSTM V24, V27, (R1) + ADD R4, R1 + VLM (R1), V24, V27 + WORD $0xe788c000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V12, V24 + WORD $0xe799d000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V13, V25 + WORD $0xe7aae000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V14, V26 + WORD $0xe7bbf000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V15, V27 + VSTM V24, V27, (R1) + RET + TEXT ·maxVXE(SB), NOSPLIT, $0-24 MOVD a+0(FP), R1 MOVD n+8(FP), R2 diff --git a/internal/kernels/vx_s390x_test.go b/internal/kernels/vx_s390x_test.go index 020d7e0..dc5482f 100644 --- a/internal/kernels/vx_s390x_test.go +++ b/internal/kernels/vx_s390x_test.go @@ -17,6 +17,17 @@ func TestVXDetected(t *testing.T) { }) } +// TestGemm32NeedsVXE: without the vector-enhancements facility 1 there is no +// float32 kernel, and HasGemm32 sends float32 products to the float64 GEMM. +func TestGemm32NeedsVXE(t *testing.T) { + if newGemm32(false) != nil { + t.Fatal("a float32 GEMM without VXE") + } + if g := newGemm32(true); g == nil || g.mr != 4 || g.nr != 16 { + t.Fatalf("newGemm32(true) = %+v", g) + } +} + // TestVXEDetected runs the max/min checks with the vector kernel forced off // too, and logs which path this machine takes. func TestVXEDetected(t *testing.T) { From bffa5cdc0d5007940844188e9191181bc4f86ce8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 30 Oct 2026 21:59:29 +0100 Subject: [PATCH 2/2] docs: s390x float32 GEMM measurements and audit entry Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- SECURITY.md | 7 +++++++ docs/perf.md | 16 +++++++++++++++- docs/plan-ndarray.md | 5 +++-- linalg.go | 2 +- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cc7c009..36a6e48 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every 64-bit target, and `Max`/`Min` on every 64-bit target, bit-identical to its scalar loops. Its `MatMul` runs on a native float32 GEMM on arm64 (NEON), - amd64 with FMA (AVX2) and ppc64le (VSX), accumulating in float32 like NumPy's sgemm, and + amd64 with FMA (AVX2), ppc64le (VSX) and s390x (z14 vector), accumulating in float32 like NumPy's sgemm, and elsewhere on the float64 GEMM (accumulated in float64, rounded once) ([docs/design-dtypes.md](docs/design-dtypes.md)). - **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/ @@ -155,7 +155,7 @@ and `Max` by 92–96% ([docs/perf.md](docs/perf.md), *SIMD coverage*). It is a > pair Go supports; v0.1.0 and v0.2.0 were also run on real amd64, arm64, > ppc64le, riscv64 and loong64 hardware, v0.3.0 on real amd64, arm64, ppc64le > and loong64. See **[docs/plan-ndarray.md](docs/plan-ndarray.md)** -> for the roadmap (a native float32 GEMM on s390x, riscv64 and loong64; +> for the roadmap (a native float32 GEMM on riscv64 and loong64; > linear algebra decompositions). ## Why this module? diff --git a/SECURITY.md b/SECURITY.md index e2aa53a..c43c2b0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,13 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-30 (v0.29.0).** The s390x float32 GEMM kernel +(`gemmMicro4x16VXE`) reads `kc·4` and `kc·16` packed floats and one 4×16 +tile of C. Its wrapper bounds-checks all three before the call, behind the +`useVXE` gate. It passed the kernel suite (fence included) and the root +suite on a real IBM Z with VXE, and a sabotaged kernel failed CI's s390x +lane. + **2026-10-27 (v0.28.0).** The ppc64le float32 GEMM kernel (`gemmMicro8x16VSX`) reads `kc·8` and `kc·16` packed floats and one 8×16 tile of C, which its wrapper bounds-checks before the call. It also reads two diff --git a/docs/perf.md b/docs/perf.md index ad33a05..71c3012 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1295,9 +1295,23 @@ The NumPy 2.5.3 wheel on that host is no reference for products: x @ x takes 14–31× slower than us, consistent with a BLAS that has no POWER9 kernels. A comparison with a tuned POWER9 OpenBLAS remains to be made. +### s390x float32 GEMM (2026-10-27) + +`gemmMicro4x16VXE` is the float64 4×8 kernel at twice the lanes: sixteen +four-lane accumulators, the B row in one `VLM`, each A value replicated with +`VLREPF`, and the z14 fused multiply-add `VFMASB`. go-asmgen encodes that +instruction, because Go's assembler has no short-format vector float op. It +needs the vector-enhancements facility 1, so it is gated on `useVXE`, like +the other float32 kernels. Without the facility, float32 products widen to +the float64 GEMM. + +| float32 `MatMul` vs v0.28.0 (`abbench`, 8 rounds, IBM Z) | 64² | 256² | 1024² | +|---|---:|---:|---:| +| widened to the float64 GEMM → native | **−58%** | **−54%** | **−50%** | + **Open defect: products are still up to 1.2× slower than OpenBLAS on one core.** The exception is float64 at 1024² on Zen 3, where we are 3% faster. -The float32 kernels for s390x, riscv64 and loong64 are still to write. +The float32 kernels for riscv64 and loong64 are still to write. **Open: the allocating operations.** `Add` is still 1.5–3.2× slower than NumPy's `a+b`, for float64 as well. The kernel is no longer the difference, diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index a001072..c409991 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -162,8 +162,9 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). GEMM since v0.21.0, 77–90% faster than the triple loop, and since v0.26.0 on a native float32 GEMM on arm64 (NEON 8×8) and amd64 with FMA (6×16), a further 53–61% faster, and since v0.28.0 on ppc64le (VSX 8×16, 37–59% - faster on a POWER9). The float32 kernels for s390x, riscv64 and loong64 - are still to write. + faster on a POWER9), and since v0.29.0 on s390x (z14 VXE 4×16, 50–58% + faster on an IBM Z). The float32 kernels for riscv64 and loong64 are + still to write. - **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic, comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`, `Where` and the per-axis reductions, in every dtype. Not done: a public diff --git a/linalg.go b/linalg.go index 677a3c2..7331de4 100644 --- a/linalg.go +++ b/linalg.go @@ -53,7 +53,7 @@ func (a *Array) MatMul(b *Array) (*Array, error) { // matmulTyped is MatMul for operands of other dtypes, in their common dtype. // // Float32 runs on the packed float32 GEMM where there is a float32 kernel -// (arm64, ppc64le, and amd64 with FMA): it accumulates in float32, as NumPy's BLAS +// (arm64, ppc64le, s390x with VXE, and amd64 with FMA): it accumulates in float32, as NumPy's BLAS // sgemm does, at twice the SIMD lanes of float64. Elsewhere it runs on the // float64 GEMM: the operands are widened (exact, O(n²)), multiplied and // accumulated in float64, and each element is rounded once to float32, which