diff --git a/docs/perf.md b/docs/perf.md index 538fafe..efc1321 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -879,10 +879,28 @@ and multiplication wrap modulo 2⁶⁴, so the order changes nothing). They also consume the slice instead of indexing `i+1…i+3`, which had left four bounds checks per iteration in the amd64 code; now there are none. -Left: ppc64le, s390x, loong64 and riscv64 run the scalar block loop. ppc64le -needs `xvaddsp`, which Go's assembler lacks: go-asmgen/asmgen#27 adds the -VSX single-precision family to its registry, with references from GNU as -and llvm-mc. +**riscv64 and loong64 (2026-10-18).** RVV: `VSETIVLI $8, E32, M2` gives +vl = 8 on every VLEN ≥ 128, and `VFADDVV` adds a block to the two-register +group. LSX: the eight accumulators sit in two 128-bit registers, and +`VFADD.S` (`VADDF`) adds a block. Each runs behind the feature bit its float64 +kernels use. Both were run on the real hardware: the kernel tests passed +there, and a sabotaged kernel failed them, so the kernel path is the one +taken. Same A/B protocol, idle hosts: + +| float32 `Sum`, 2¹⁴ / 2²⁰ | change | controls | +|---|---:|---| +| SpacemiT X60 (cfarm95, RVV, VLEN 256) | −30% / −41% | float64, int32 ~ | +| Loongson 3C5000L (cfarm401, LSX) | −24% / −24% | float64, int32 ~ | + +The first loong64 kernel used one 256-bit LASX register for all eight +accumulators, and was **26–47% slower** than the scalar loop. Every block's +addition waited on the previous one, a single dependency chain, where the +scalar loop has eight. Two registers give two chains, as SSE2 does on amd64. + +Left: ppc64le and s390x run the scalar block loop. Go's assembler lacks +`xvaddsp` (ppc64) and `vfasb` (s390x). go-asmgen/asmgen#27 adds the VSX +single-precision family to go-asmgen's registry, with references from GNU as +and llvm-mc, and needs a cmd/asm patch of its own. ## SIMD coverage diff --git a/internal/kernels/asmgen/loong64/gen.go b/internal/kernels/asmgen/loong64/gen.go index 21e6cb4..6d5d157 100644 --- a/internal/kernels/asmgen/loong64/gen.go +++ b/internal/kernels/asmgen/loong64/gen.go @@ -39,6 +39,7 @@ func main() { f.Add(binKernel(k.name, k.vec, k.sca)) } f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_loong64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -206,3 +207,32 @@ func gemmKernel() *emit.Function { b.Ret() return b.Func() } + +// blocks8Kernel builds blocks8LSX(r, a *float32, blocks int): for each of +// the blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's +// pairwise-sum block). The eight accumulators are split over two 128-bit LSX +// registers, V0 and V1, and VADDF (vfadd.s) is one IEEE float32 addition per +// lane, so the result is the scalar loop's bit for bit. blocks >= 1. +// +// Not one 256-bit LASX register: that version kept every block's addition +// waiting on the previous one, a single dependency chain, and on a Loongson +// 3C5000L it was 26-47% SLOWER than the scalar loop's eight chains. Two +// registers, two chains (as SSE2 on amd64): 24% faster. +func blocks8Kernel() *emit.Function { + sig := loong64.Layout([]string{"r", "a", "blocks"}, + []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil) + b := loong64.NewFunc("blocks8LSX", sig, 0) + b.LoadArg("r", "R4").LoadArg("a", "R5").LoadArg("blocks", "R6") + b.Raw("VMOVQ 0(R4), V0").Raw("VMOVQ 16(R4), V1") + b.Label("b8loop"). + Raw("VMOVQ 0(R5), V2"). + Raw("VMOVQ 16(R5), V3"). + Raw("VADDF V2, V0, V0"). + Raw("VADDF V3, V1, V1"). + Raw("ADDV $32, R5"). + Raw("ADDV $-1, R6"). + Raw("BNE R6, b8loop") + b.Raw("VMOVQ V0, 0(R4)").Raw("VMOVQ V1, 16(R4)") + b.Ret() + return b.Func() +} diff --git a/internal/kernels/asmgen/riscv64/gen.go b/internal/kernels/asmgen/riscv64/gen.go index cbc6fa1..4913f50 100644 --- a/internal/kernels/asmgen/riscv64/gen.go +++ b/internal/kernels/asmgen/riscv64/gen.go @@ -36,6 +36,7 @@ func main() { f.Add(binKernel(k.name, k.op)) } f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_riscv64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -168,3 +169,26 @@ func gemmKernel() *emit.Function { } return b.Ret().Func() } + +// blocks8Kernel builds blocks8RVV(r, a *float32, blocks int): for each of the +// blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's +// pairwise-sum block). VSETIVLI $8 with E32 and M2 gives vl = 8 on every +// VLEN >= 128, the V extension's minimum; the eight accumulators live in the +// register group V2-V3 and VFADDVV is one IEEE float32 addition per lane, so +// the result is the scalar loop's bit for bit. blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := riscv64.Layout([]string{"r", "a", "blocks"}, + []riscv64.Type{riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) + b := riscv64.NewFunc("blocks8RVV", sig, 0) + b.LoadArg("r", "X10").LoadArg("a", "X11").LoadArg("blocks", "X12") + b.Raw("VSETIVLI $8, E32, M2, TA, MA, X15"). + Raw("VLE32V (X10), V2"). + Raw("b8loop:"). + Raw("VLE32V (X11), V4"). + Raw("VFADDVV V4, V2, V2"). // V2 = V2 + V4 + Raw("ADD $32, X11"). + Raw("ADD $-1, X12"). + Raw("BNEZ X12, b8loop"). + Raw("VSE32V V2, (X10)") + return b.Ret().Func() +} diff --git a/internal/kernels/blocks8_generic.go b/internal/kernels/blocks8_generic.go index 28b2307..489ac6c 100644 --- a/internal/kernels/blocks8_generic.go +++ b/internal/kernels/blocks8_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 +//go:build !amd64 && !arm64 && !loong64 && !riscv64 package kernels diff --git a/internal/kernels/blocks8_loong64.go b/internal/kernels/blocks8_loong64.go new file mode 100644 index 0000000..adc475e --- /dev/null +++ b/internal/kernels/blocks8_loong64.go @@ -0,0 +1,17 @@ +package kernels + +// addBlocks8F32 runs the LSX kernel (sum_loong64.s), the eight accumulators +// in two 128-bit registers, when the CPU has LASX (which every LASX CPU's LSX +// comes with); the scalar loop otherwise. len(a) is a positive multiple of 8 +// (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + if !useLASX { + addBlocks8F32Go(r, a) + return + } + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8LSX(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8LSX(r, a *float32, blocks int) diff --git a/internal/kernels/blocks8_riscv64.go b/internal/kernels/blocks8_riscv64.go new file mode 100644 index 0000000..fc5943a --- /dev/null +++ b/internal/kernels/blocks8_riscv64.go @@ -0,0 +1,17 @@ +package kernels + +// addBlocks8F32 runs the RVV kernel (sum_riscv64.s), the eight accumulators +// in a two-register group at vl = 8, when the CPU has the V extension; the +// scalar loop otherwise. len(a) is a positive multiple of 8 (AddBlocks8F32 +// checks). +func addBlocks8F32(r *[8]float32, a []float32) { + if !useRVV { + addBlocks8F32Go(r, a) + return + } + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8RVV(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8RVV(r, a *float32, blocks int) diff --git a/internal/kernels/sum_loong64.s b/internal/kernels/sum_loong64.s index 5c4195a..4a88d40 100644 --- a/internal/kernels/sum_loong64.s +++ b/internal/kernels/sum_loong64.s @@ -426,3 +426,21 @@ gstore: XVMOVQ X17, 32(R8) RET +TEXT ·blocks8LSX(SB), NOSPLIT, $0-24 + MOVV r+0(FP), R4 + MOVV a+8(FP), R5 + MOVV blocks+16(FP), R6 + VMOVQ 0(R4), V0 + VMOVQ 16(R4), V1 +b8loop: + VMOVQ 0(R5), V2 + VMOVQ 16(R5), V3 + VADDF V2, V0, V0 + VADDF V3, V1, V1 + ADDV $32, R5 + ADDV $-1, R6 + BNE R6, b8loop + VMOVQ V0, 0(R4) + VMOVQ V1, 16(R4) + RET + diff --git a/internal/kernels/sum_riscv64.s b/internal/kernels/sum_riscv64.s index 920fe6b..07fa8d2 100644 --- a/internal/kernels/sum_riscv64.s +++ b/internal/kernels/sum_riscv64.s @@ -198,3 +198,18 @@ TEXT ·gemmMicro6x8RVV(SB), NOSPLIT, $0-40 VSE64V V28, (X13) RET +TEXT ·blocks8RVV(SB), NOSPLIT, $0-24 + MOV r+0(FP), X10 + MOV a+8(FP), X11 + MOV blocks+16(FP), X12 + VSETIVLI $8, E32, M2, TA, MA, X15 + VLE32V (X10), V2 + b8loop: + VLE32V (X11), V4 + VFADDVV V4, V2, V2 + ADD $32, X11 + ADD $-1, X12 + BNEZ X12, b8loop + VSE32V V2, (X10) + RET +