From 6dd25796b1162e2ec97239a7c936394fcc4f1ad8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 16:04:41 +0200 Subject: [PATCH 1/3] perf: float32 pairwise sum blocks on RVV (riscv64) and LASX (loong64) AddBlocks8F32 gains two kernels, each behind the feature bit the float64 kernels already use (useRVV, useLASX), the scalar loop otherwise: - riscv64: VSETIVLI $8, E32, M2 gives vl = 8 on every VLEN >= 128; the eight accumulators in the group V2-V3, VFADDVV per block. - loong64: the eight accumulators fill one 256-bit register, XVADDF (xvfadd.s) per block. One IEEE float32 addition per lane per block, in order: float32 sums are unchanged bit for bit. Both are spelled by Go's own assembler. Run on real hardware: the kernel tests (bit-identity, refusals, fence) and the float32 sum tests pass on cfarm95 (SpacemiT X60, RVV, VLEN 256) and cfarm401 (Loongson, LASX); a sabotaged kernel fails the bit-identity test on both, so the kernel path is the one taken there. All seven targets build. Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/loong64/gen.go | 23 +++++++++++++++++++++++ internal/kernels/asmgen/riscv64/gen.go | 24 ++++++++++++++++++++++++ internal/kernels/blocks8_generic.go | 2 +- internal/kernels/blocks8_loong64.go | 16 ++++++++++++++++ internal/kernels/blocks8_riscv64.go | 17 +++++++++++++++++ internal/kernels/sum_loong64.s | 14 ++++++++++++++ internal/kernels/sum_riscv64.s | 15 +++++++++++++++ 7 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 internal/kernels/blocks8_loong64.go create mode 100644 internal/kernels/blocks8_riscv64.go diff --git a/internal/kernels/asmgen/loong64/gen.go b/internal/kernels/asmgen/loong64/gen.go index 21e6cb4..5f86139 100644 --- a/internal/kernels/asmgen/loong64/gen.go +++ b/internal/kernels/asmgen/loong64/gen.go @@ -39,6 +39,7 @@ func main() { f.Add(binKernel(k.name, k.vec, k.sca)) } f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_loong64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -206,3 +207,25 @@ func gemmKernel() *emit.Function { b.Ret() return b.Func() } + +// blocks8Kernel builds blocks8LASX(r, a *float32, blocks int): for each of +// the blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's +// pairwise-sum block). The eight accumulators fill one 256-bit register, X0; +// XVADDF (xvfadd.s) is one IEEE float32 addition per lane, so the result is +// the scalar loop's bit for bit. blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := loong64.Layout([]string{"r", "a", "blocks"}, + []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil) + b := loong64.NewFunc("blocks8LASX", sig, 0) + b.LoadArg("r", "R4").LoadArg("a", "R5").LoadArg("blocks", "R6") + b.Raw("XVMOVQ 0(R4), X0") + b.Label("b8loop"). + Raw("XVMOVQ 0(R5), X1"). + Raw("XVADDF X1, X0, X0"). + Raw("ADDV $32, R5"). + Raw("ADDV $-1, R6"). + Raw("BNE R6, b8loop") + b.Raw("XVMOVQ X0, 0(R4)") + b.Ret() + return b.Func() +} diff --git a/internal/kernels/asmgen/riscv64/gen.go b/internal/kernels/asmgen/riscv64/gen.go index cbc6fa1..4913f50 100644 --- a/internal/kernels/asmgen/riscv64/gen.go +++ b/internal/kernels/asmgen/riscv64/gen.go @@ -36,6 +36,7 @@ func main() { f.Add(binKernel(k.name, k.op)) } f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_riscv64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -168,3 +169,26 @@ func gemmKernel() *emit.Function { } return b.Ret().Func() } + +// blocks8Kernel builds blocks8RVV(r, a *float32, blocks int): for each of the +// blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's +// pairwise-sum block). VSETIVLI $8 with E32 and M2 gives vl = 8 on every +// VLEN >= 128, the V extension's minimum; the eight accumulators live in the +// register group V2-V3 and VFADDVV is one IEEE float32 addition per lane, so +// the result is the scalar loop's bit for bit. blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := riscv64.Layout([]string{"r", "a", "blocks"}, + []riscv64.Type{riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) + b := riscv64.NewFunc("blocks8RVV", sig, 0) + b.LoadArg("r", "X10").LoadArg("a", "X11").LoadArg("blocks", "X12") + b.Raw("VSETIVLI $8, E32, M2, TA, MA, X15"). + Raw("VLE32V (X10), V2"). + Raw("b8loop:"). + Raw("VLE32V (X11), V4"). + Raw("VFADDVV V4, V2, V2"). // V2 = V2 + V4 + Raw("ADD $32, X11"). + Raw("ADD $-1, X12"). + Raw("BNEZ X12, b8loop"). + Raw("VSE32V V2, (X10)") + return b.Ret().Func() +} diff --git a/internal/kernels/blocks8_generic.go b/internal/kernels/blocks8_generic.go index 28b2307..489ac6c 100644 --- a/internal/kernels/blocks8_generic.go +++ b/internal/kernels/blocks8_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 +//go:build !amd64 && !arm64 && !loong64 && !riscv64 package kernels diff --git a/internal/kernels/blocks8_loong64.go b/internal/kernels/blocks8_loong64.go new file mode 100644 index 0000000..abb5838 --- /dev/null +++ b/internal/kernels/blocks8_loong64.go @@ -0,0 +1,16 @@ +package kernels + +// addBlocks8F32 runs the LASX kernel (sum_loong64.s), the eight accumulators +// in one 256-bit register, when the CPU has LASX; the scalar loop otherwise. +// len(a) is a positive multiple of 8 (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + if !useLASX { + addBlocks8F32Go(r, a) + return + } + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8LASX(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8LASX(r, a *float32, blocks int) diff --git a/internal/kernels/blocks8_riscv64.go b/internal/kernels/blocks8_riscv64.go new file mode 100644 index 0000000..fc5943a --- /dev/null +++ b/internal/kernels/blocks8_riscv64.go @@ -0,0 +1,17 @@ +package kernels + +// addBlocks8F32 runs the RVV kernel (sum_riscv64.s), the eight accumulators +// in a two-register group at vl = 8, when the CPU has the V extension; the +// scalar loop otherwise. len(a) is a positive multiple of 8 (AddBlocks8F32 +// checks). +func addBlocks8F32(r *[8]float32, a []float32) { + if !useRVV { + addBlocks8F32Go(r, a) + return + } + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8RVV(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8RVV(r, a *float32, blocks int) diff --git a/internal/kernels/sum_loong64.s b/internal/kernels/sum_loong64.s index 5c4195a..9049d0f 100644 --- a/internal/kernels/sum_loong64.s +++ b/internal/kernels/sum_loong64.s @@ -426,3 +426,17 @@ gstore: XVMOVQ X17, 32(R8) RET +TEXT ·blocks8LASX(SB), NOSPLIT, $0-24 + MOVV r+0(FP), R4 + MOVV a+8(FP), R5 + MOVV blocks+16(FP), R6 + XVMOVQ 0(R4), X0 +b8loop: + XVMOVQ 0(R5), X1 + XVADDF X1, X0, X0 + ADDV $32, R5 + ADDV $-1, R6 + BNE R6, b8loop + XVMOVQ X0, 0(R4) + RET + diff --git a/internal/kernels/sum_riscv64.s b/internal/kernels/sum_riscv64.s index 920fe6b..07fa8d2 100644 --- a/internal/kernels/sum_riscv64.s +++ b/internal/kernels/sum_riscv64.s @@ -198,3 +198,18 @@ TEXT ·gemmMicro6x8RVV(SB), NOSPLIT, $0-40 VSE64V V28, (X13) RET +TEXT ·blocks8RVV(SB), NOSPLIT, $0-24 + MOV r+0(FP), X10 + MOV a+8(FP), X11 + MOV blocks+16(FP), X12 + VSETIVLI $8, E32, M2, TA, MA, X15 + VLE32V (X10), V2 + b8loop: + VLE32V (X11), V4 + VFADDVV V4, V2, V2 + ADD $32, X11 + ADD $-1, X12 + BNEZ X12, b8loop + VSE32V V2, (X10) + RET + From 5be2bf813ad4313557108b611664599e1021c331 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 16:15:38 +0200 Subject: [PATCH 2/3] kernels/loong64: the float32 block kernel on two LSX registers, not one LASX The 256-bit version kept the eight accumulators in one register, a single dependency chain, and was 26-47% slower than the scalar loop on a Loongson 3C5000L. Two 128-bit registers, two chains: -24% at 2^14 and 2^20 there, bit-identity test passing on the hardware. Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/loong64/gen.go | 25 ++++++++++++++++--------- internal/kernels/blocks8_loong64.go | 11 ++++++----- internal/kernels/sum_loong64.s | 14 +++++++++----- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/internal/kernels/asmgen/loong64/gen.go b/internal/kernels/asmgen/loong64/gen.go index 5f86139..6d5d157 100644 --- a/internal/kernels/asmgen/loong64/gen.go +++ b/internal/kernels/asmgen/loong64/gen.go @@ -208,24 +208,31 @@ func gemmKernel() *emit.Function { return b.Func() } -// blocks8Kernel builds blocks8LASX(r, a *float32, blocks int): for each of +// blocks8Kernel builds blocks8LSX(r, a *float32, blocks int): for each of // the blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's -// pairwise-sum block). The eight accumulators fill one 256-bit register, X0; -// XVADDF (xvfadd.s) is one IEEE float32 addition per lane, so the result is -// the scalar loop's bit for bit. blocks >= 1. +// pairwise-sum block). The eight accumulators are split over two 128-bit LSX +// registers, V0 and V1, and VADDF (vfadd.s) is one IEEE float32 addition per +// lane, so the result is the scalar loop's bit for bit. blocks >= 1. +// +// Not one 256-bit LASX register: that version kept every block's addition +// waiting on the previous one, a single dependency chain, and on a Loongson +// 3C5000L it was 26-47% SLOWER than the scalar loop's eight chains. Two +// registers, two chains (as SSE2 on amd64): 24% faster. func blocks8Kernel() *emit.Function { sig := loong64.Layout([]string{"r", "a", "blocks"}, []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil) - b := loong64.NewFunc("blocks8LASX", sig, 0) + b := loong64.NewFunc("blocks8LSX", sig, 0) b.LoadArg("r", "R4").LoadArg("a", "R5").LoadArg("blocks", "R6") - b.Raw("XVMOVQ 0(R4), X0") + b.Raw("VMOVQ 0(R4), V0").Raw("VMOVQ 16(R4), V1") b.Label("b8loop"). - Raw("XVMOVQ 0(R5), X1"). - Raw("XVADDF X1, X0, X0"). + Raw("VMOVQ 0(R5), V2"). + Raw("VMOVQ 16(R5), V3"). + Raw("VADDF V2, V0, V0"). + Raw("VADDF V3, V1, V1"). Raw("ADDV $32, R5"). Raw("ADDV $-1, R6"). Raw("BNE R6, b8loop") - b.Raw("XVMOVQ X0, 0(R4)") + b.Raw("VMOVQ V0, 0(R4)").Raw("VMOVQ V1, 16(R4)") b.Ret() return b.Func() } diff --git a/internal/kernels/blocks8_loong64.go b/internal/kernels/blocks8_loong64.go index abb5838..adc475e 100644 --- a/internal/kernels/blocks8_loong64.go +++ b/internal/kernels/blocks8_loong64.go @@ -1,16 +1,17 @@ package kernels -// addBlocks8F32 runs the LASX kernel (sum_loong64.s), the eight accumulators -// in one 256-bit register, when the CPU has LASX; the scalar loop otherwise. -// len(a) is a positive multiple of 8 (AddBlocks8F32 checks). +// addBlocks8F32 runs the LSX kernel (sum_loong64.s), the eight accumulators +// in two 128-bit registers, when the CPU has LASX (which every LASX CPU's LSX +// comes with); the scalar loop otherwise. len(a) is a positive multiple of 8 +// (AddBlocks8F32 checks). func addBlocks8F32(r *[8]float32, a []float32) { if !useLASX { addBlocks8F32Go(r, a) return } _ = a[len(a)-1] // the kernel reads len(a) of a - blocks8LASX(&r[0], &a[0], len(a)/8) + blocks8LSX(&r[0], &a[0], len(a)/8) } //go:noescape -func blocks8LASX(r, a *float32, blocks int) +func blocks8LSX(r, a *float32, blocks int) diff --git a/internal/kernels/sum_loong64.s b/internal/kernels/sum_loong64.s index 9049d0f..4a88d40 100644 --- a/internal/kernels/sum_loong64.s +++ b/internal/kernels/sum_loong64.s @@ -426,17 +426,21 @@ gstore: XVMOVQ X17, 32(R8) RET -TEXT ·blocks8LASX(SB), NOSPLIT, $0-24 +TEXT ·blocks8LSX(SB), NOSPLIT, $0-24 MOVV r+0(FP), R4 MOVV a+8(FP), R5 MOVV blocks+16(FP), R6 - XVMOVQ 0(R4), X0 + VMOVQ 0(R4), V0 + VMOVQ 16(R4), V1 b8loop: - XVMOVQ 0(R5), X1 - XVADDF X1, X0, X0 + VMOVQ 0(R5), V2 + VMOVQ 16(R5), V3 + VADDF V2, V0, V0 + VADDF V3, V1, V1 ADDV $32, R5 ADDV $-1, R6 BNE R6, b8loop - XVMOVQ X0, 0(R4) + VMOVQ V0, 0(R4) + VMOVQ V1, 16(R4) RET From 4a3486a23489835b97a55235bf9333d1255acdcc Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 16:16:00 +0200 Subject: [PATCH 3/3] docs/perf.md: float32 blocks on RVV and LSX, measured on an X60 and a 3C5000L Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/perf.md b/docs/perf.md index 538fafe..efc1321 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -879,10 +879,28 @@ and multiplication wrap modulo 2⁶⁴, so the order changes nothing). They also consume the slice instead of indexing `i+1…i+3`, which had left four bounds checks per iteration in the amd64 code; now there are none. -Left: ppc64le, s390x, loong64 and riscv64 run the scalar block loop. ppc64le -needs `xvaddsp`, which Go's assembler lacks: go-asmgen/asmgen#27 adds the -VSX single-precision family to its registry, with references from GNU as -and llvm-mc. +**riscv64 and loong64 (2026-10-18).** RVV: `VSETIVLI $8, E32, M2` gives +vl = 8 on every VLEN ≥ 128, and `VFADDVV` adds a block to the two-register +group. LSX: the eight accumulators sit in two 128-bit registers, and +`VFADD.S` (`VADDF`) adds a block. Each runs behind the feature bit its float64 +kernels use. Both were run on the real hardware: the kernel tests passed +there, and a sabotaged kernel failed them, so the kernel path is the one +taken. Same A/B protocol, idle hosts: + +| float32 `Sum`, 2¹⁴ / 2²⁰ | change | controls | +|---|---:|---| +| SpacemiT X60 (cfarm95, RVV, VLEN 256) | −30% / −41% | float64, int32 ~ | +| Loongson 3C5000L (cfarm401, LSX) | −24% / −24% | float64, int32 ~ | + +The first loong64 kernel used one 256-bit LASX register for all eight +accumulators, and was **26–47% slower** than the scalar loop. Every block's +addition waited on the previous one, a single dependency chain, where the +scalar loop has eight. Two registers give two chains, as SSE2 does on amd64. + +Left: ppc64le and s390x run the scalar block loop. Go's assembler lacks +`xvaddsp` (ppc64) and `vfasb` (s390x). go-asmgen/asmgen#27 adds the VSX +single-precision family to go-asmgen's registry, with references from GNU as +and llvm-mc, and needs a cmd/asm patch of its own. ## SIMD coverage