Skip to content

Commit 8388dc6

Browse files
tannevaledclaude
andauthored
perf: float32 Sqrt kernels on six targets; float32 survey against NumPy (#61)
Against NumPy 2.5.3 on one POWER9 core, float32 Sqrt was 6.8-7.5x slower: it called math.Sqrt through a function value per element. kernels.Sqrt32 runs SQRTPS (amd64), NEON .S4 (arm64), xvsqrtsp (ppc64le), vfsqsb (s390x, z14+), LSX vfsqrt.s (loong64) and RVV E32 (riscv64), 16 floats per iteration. Each is the correctly rounded float32 square root, so float32(math.Sqrt(float64(x))) bit for bit. POWER9, abbench (alternated, 8 rounds, pinned core): float32 Sqrt -77% (2^14) / -83% (2^20); float64 Sqrt, Add and AddInto flat. benchmarks: DTypeSqrt, DTypeMax, DTypeMatMul. docs/perf.md records the survey: float32 Max is 11.9-15x slower than NumPy (no kernel; on ppc64le float64 Max has none either), and float32 MatMul is 9x slower than our float64 GEMM. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 2432766 commit 8388dc6

28 files changed

Lines changed: 570 additions & 7 deletions

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
complex64/complex128, chosen at run time with **NumPy 2 promotion** (`AsType`,
1616
`ZerosOf`, `FromSlice`, typed access with `Data[T]`/`Item[T]`, weak Go
1717
scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated
18-
one. Float32 has SIMD `Sum` and `Add`/`Sub`/`Mul`/`Div` on every 64-bit
19-
target, bit-identical to its scalar loops
18+
one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every
19+
64-bit target, bit-identical to its scalar loops
2020
([docs/design-dtypes.md](docs/design-dtypes.md)).
2121
- **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/
2222
`Eye`/`Identity`.
@@ -152,8 +152,8 @@ and `Max` by 92–96% ([docs/perf.md](docs/perf.md), *SIMD coverage*). It is a
152152
> pair Go supports; v0.1.0 and v0.2.0 were also run on real amd64, arm64,
153153
> ppc64le, riscv64 and loong64 hardware, v0.3.0 on real amd64, arm64, ppc64le
154154
> and loong64. See **[docs/plan-ndarray.md](docs/plan-ndarray.md)**
155-
> for the roadmap (float32 SIMD for sqrt, max/min and the products; linear
156-
> algebra decompositions).
155+
> for the roadmap (float32 SIMD for max/min and the products; linear algebra
156+
> decompositions).
157157
158158
## Why this module?
159159

‎SECURITY.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none.
6666

6767
## Audit log
6868

69+
**2026-10-20 (v0.19.0).** `kernels.Sqrt32` follows `Bin32`'s guards: it
70+
reslices `src` to `len(dst)` before the call (a short `src` panics in Go,
71+
tested), never passes an empty slice to a kernel, and keeps the LASX, V and
72+
VXE gates. The PROT_NONE fence now also covers it, and passes on Zen 3,
73+
Apple M, POWER9, IBM Z, SpacemiT X60 and Loongson 3C5000L.
74+
6975
**2026-10-18 (v0.18.0).** The float32 elementwise kernels (`kernels.Bin32`,
7076
on all six 64-bit targets) take raw pointers and a count. `Bin32` reslices `a`
7177
and `b` to `len(dst)` before the call, so a short operand panics in Go rather

‎arith.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"fmt"
55
"math/cmplx"
66
"reflect"
7+
8+
"github.com/go-ndarray/ndarray/internal/kernels"
79
)
810

911
// The operations on dtypes other than Float64. Each public method keeps its
@@ -224,7 +226,11 @@ func (a *Array) mathUnary(code unaryCode, f64 func(*Array) *Array) *Array {
224226
d := makeStore(dt, a.Size())
225227
switch s := src.(type) {
226228
case []float32:
227-
unaryFloat32(realFuncs[code], d.([]float32), s)
229+
if code == uSqrt {
230+
kernels.Sqrt32(d.([]float32), s) // the same correctly rounded result
231+
} else {
232+
unaryFloat32(realFuncs[code], d.([]float32), s)
233+
}
228234
case []complex64:
229235
unaryComplex(a.complexFunc(code), d.([]complex64), s)
230236
case []complex128:

‎benchmarks/dtype_bench_test.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,53 @@ func BenchmarkDTypeAdd(b *testing.B) {
4242
}
4343
}
4444

45+
func BenchmarkDTypeSqrt(b *testing.B) {
46+
for _, dt := range []nd.DType{nd.Float64, nd.Float32} {
47+
for _, n := range []int{1 << 14, 1 << 20} {
48+
x := vec(n).AsType(dt)
49+
b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) {
50+
b.SetBytes(int64(n * dt.Size()))
51+
for i := 0; i < b.N; i++ {
52+
_ = x.Sqrt()
53+
}
54+
})
55+
}
56+
}
57+
}
58+
59+
func BenchmarkDTypeMax(b *testing.B) {
60+
for _, dt := range []nd.DType{nd.Float64, nd.Float32} {
61+
for _, n := range []int{1 << 14, 1 << 20} {
62+
x := vec(n).AsType(dt)
63+
b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) {
64+
b.SetBytes(int64(n * dt.Size()))
65+
for i := 0; i < b.N; i++ {
66+
_, _ = x.Max()
67+
}
68+
})
69+
}
70+
}
71+
}
72+
73+
// BenchmarkDTypeMatMul is a square product, 2·n³ flops per op.
74+
func BenchmarkDTypeMatMul(b *testing.B) {
75+
for _, dt := range []nd.DType{nd.Float64, nd.Float32} {
76+
for _, n := range []int{64, 256} {
77+
x, err := vec(n * n).AsType(dt).Reshape(n, n)
78+
if err != nil {
79+
b.Fatal(err)
80+
}
81+
b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) {
82+
for i := 0; i < b.N; i++ {
83+
if _, err := x.MatMul(x); err != nil {
84+
b.Fatal(err)
85+
}
86+
}
87+
})
88+
}
89+
}
90+
}
91+
4592
// BenchmarkDTypeAddInto is Add without the allocation, the analogue of
4693
// np.add(x, y, out=out): it measures the kernel, where BenchmarkDTypeAdd also
4794
// measures allocating and clearing the result.

‎docs/perf.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -976,6 +976,48 @@ POWER9, one core, against NumPy 2.5.3:
976976
float64 `AddInto` against `np.add(out=)`: 5.70 against 7.75 µs (2¹⁴), and
977977
724 against 768 µs (2²⁰).
978978

979+
### float32 survey against NumPy, and `Sqrt` (2026-10-20)
980+
981+
The rest of the float32 surface, on the same POWER9 core against NumPy 2.5.3:
982+
983+
| | ours | NumPy | |
984+
|---|---:|---:|---|
985+
| float32 `Max`, 2¹⁴ / 2²⁰ | 84.7 µs / 4.86 ms | 7.13 µs / 0.324 ms | **11.9× / 15× slower** |
986+
| float64 `Max`, 2¹⁴ / 2²⁰ | 51.6 µs / 3.33 ms | 11.8 µs / 0.615 ms | 4.4× / 5.4× slower |
987+
| float32 `Sqrt`, 2¹⁴ / 2²⁰ | 100 µs / 6.3 ms | 14.7 µs / 0.835 ms | 6.8× / 7.5× slower |
988+
| float64 `Sqrt`, 2¹⁴ / 2²⁰ | 52.5 µs / 2.7 ms | 37.4 µs / 2.3 ms | 1.4× / 1.2× slower |
989+
| `MatMul` 256², float32 / float64 | 20.4 / 2.25 ms | 30.4 / 29.8 ms | NumPy here has no BLAS |
990+
991+
Read each row against the one below it:
992+
993+
- float32 `Max` has no kernel anywhere, and on ppc64le neither has float64:
994+
`xvmaxdp` does not propagate NaN as NumPy does.
995+
- float32 `Sqrt` called `math.Sqrt` through a function value for every
996+
element.
997+
- float32 `MatMul` is 9× slower than our own float64 `MatMul`: float32 has no
998+
packed GEMM.
999+
1000+
`Sqrt` is fixed here. `kernels.Sqrt32` runs `SQRTPS`, NEON `.S4`,
1001+
`xvsqrtsp`, `vfsqsb` (z14), LSX `vfsqrt.s` or RVV E32. Each is the correctly
1002+
rounded float32 square root, so `float32(math.Sqrt(float64(x)))` bit for bit:
1003+
float64 carries more than twice float32's precision.
1004+
1005+
The kernel tests (special values, every length to 70, in place) and the
1006+
PROT_NONE fence pass on Zen 3, Apple M, POWER9, IBM Z, SpacemiT X60 and
1007+
Loongson 3C5000L. Measured with fleettools' `abbench` (base and head
1008+
alternated, 8 rounds, pinned core, `--max-busy 15`):
1009+
1010+
| float32 `Sqrt`, POWER9 | before | after | NumPy |
1011+
|---|---:|---:|---:|
1012+
| 2¹⁴ | 102.9 µs | **23.4 µs** (−77%) | 14.7 µs |
1013+
| 2²⁰ | 6.21 ms | **1.08 ms** (−83%) | 0.835 ms |
1014+
1015+
The controls (float64 `Sqrt`, `Add`, `AddInto`) are flat. The gap left to
1016+
NumPy is the allocation, the open defect below.
1017+
1018+
Still to do, in this order: `Max`/`Min` (float32 everywhere, float64 on
1019+
ppc64le, loong64 and riscv64), then a float32 GEMM.
1020+
9791021
**Open defect: the allocating operations.** `Add` is still 1.5–3.2× slower
9801022
than NumPy's `a+b`, for float64 as well (21.9 against 8.48 µs at 2¹⁴). The
9811023
kernel is no longer the difference, as `AddInto` shows; the allocation is.

‎docs/plan-ndarray.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,10 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`).
156156
with NumPy 2.x promotion; see [design-dtypes.md](design-dtypes.md). The
157157
`fft` bridge shipped in v0.10.0 (package `fft`, over go-fft). float32 has
158158
SIMD kernels on all six 64-bit targets for `Sum` (v0.14.0–v0.17.0) and for
159-
`Add`/`Sub`/`Mul`/`Div` (v0.18.0). Still pure Go for float32: `Sqrt`,
160-
`Max`/`Min`, `Dot`/`MatMul` and the transcendental ufuncs.
159+
`Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go
160+
for float32: `Max`/`Min` (11.9–15× slower than NumPy on a POWER9),
161+
`Dot`/`MatMul` (9× slower than our float64 GEMM) and the transcendental
162+
ufuncs.
161163
- **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic,
162164
comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`,
163165
`Where` and the per-axis reductions, in every dtype. Not done: a public

‎internal/kernels/asmgen/amd64/gen.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ func main() {
5454
f.Add(bin32Kernel("sub32SSE2", "SUBPS", "SUBSS"))
5555
f.Add(bin32Kernel("mul32SSE2", "MULPS", "MULSS"))
5656
f.Add(bin32Kernel("div32SSE2", "DIVPS", "DIVSS"))
57+
f.Add(sqrt32Kernel())
5758

5859
if err := os.WriteFile("sum_amd64.s", []byte(f.String()), 0o644); err != nil {
5960
fmt.Fprintln(os.Stderr, err)
@@ -329,6 +330,37 @@ func binKernel(name, vec, sca string) *emit.Function {
329330
return b.Func()
330331
}
331332

333+
// sqrt32Kernel builds sqrt32SSE2(dst, src *float32, n int): SQRTPS over 16
334+
// floats per iteration, SQRTSS for the tail. Both are the correctly rounded
335+
// IEEE float32 square root, which is float32(math.Sqrt(float64(x))) bit for
336+
// bit (float64 carries more than twice float32's precision).
337+
func sqrt32Kernel() *emit.Function {
338+
sig := amd64.Layout(
339+
[]string{"dst", "src", "n"},
340+
[]amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64},
341+
nil, nil,
342+
)
343+
b := amd64.NewFunc("sqrt32SSE2", sig, 0)
344+
b.LoadArg("dst", "DI").LoadArg("src", "SI").LoadArg("n", "CX")
345+
b.Raw("qblock:").Raw("CMPQ CX, $16").Raw("JL qtail")
346+
for r := 0; r < 4; r++ {
347+
b.Raw("MOVUPS %d(SI), X%d", 16*r, r)
348+
}
349+
for r := 0; r < 4; r++ {
350+
b.Raw("SQRTPS X%d, X%d", r, r)
351+
}
352+
for r := 0; r < 4; r++ {
353+
b.Raw("MOVUPS X%d, %d(DI)", r, 16*r)
354+
}
355+
b.Raw("ADDQ $64, SI").Raw("ADDQ $64, DI").Raw("SUBQ $16, CX").Raw("JMP qblock").
356+
Raw("qtail:").Raw("TESTQ CX, CX").Raw("JZ qdone").
357+
Raw("MOVSS (SI), X0").Raw("SQRTSS X0, X0").Raw("MOVSS X0, (DI)").
358+
Raw("ADDQ $4, SI").Raw("ADDQ $4, DI").Raw("DECQ CX").Raw("JMP qtail").
359+
Raw("qdone:")
360+
b.Ret()
361+
return b.Func()
362+
}
363+
332364
// bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
333365
// 16 floats per iteration over four XMM registers with the packed op, then a
334366
// scalar tail with sca. Each lane is one IEEE float32 operation, so the result

‎internal/kernels/asmgen/arm64/gen.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ func main() {
5151
f.Add(bin32Kernel("sub32NEON", "VFSUB", "FSUBS"))
5252
f.Add(bin32Kernel("mul32NEON", "VFMUL", "FMULS"))
5353
f.Add(bin32Kernel("div32NEON", "VFDIV", "FDIVS"))
54+
f.Add(sqrt32Kernel())
5455
f.Add(extremumKernel("maxNEON", "VFMAX", "FMAXD"))
5556
f.Add(extremumKernel("minNEON", "VFMIN", "FMIND"))
5657
f.Add(gemmKernel())
@@ -156,6 +157,31 @@ func binKernel(name, vop, sop string) *emit.Function {
156157
return b.Func()
157158
}
158159

160+
// sqrt32Kernel builds sqrt32NEON(dst, src *float32, n int): VFSQRT on .S4
161+
// over 16 floats per iteration, FSQRTS for the tail; the correctly rounded
162+
// float32 square root, bit-identical to float32(math.Sqrt(float64(x))).
163+
func sqrt32Kernel() *emit.Function {
164+
sig := arm64.Layout(
165+
[]string{"dst", "src", "n"},
166+
[]arm64.Type{arm64.Ptr, arm64.Ptr, arm64.Int64},
167+
nil, nil,
168+
)
169+
b := arm64.NewFunc("sqrt32NEON", sig, 0)
170+
b.LoadArg("dst", "R0").LoadArg("src", "R1").LoadArg("n", "R2").
171+
Raw("block:").Raw("CMP $16, R2").Raw("BLT tail").
172+
Raw("VLD1.P 64(R1), [V0.S4, V1.S4, V2.S4, V3.S4]")
173+
for r := 0; r < 4; r++ {
174+
b.Raw(fmt.Sprintf("VFSQRT V%d.S4, V%d.S4", r, r))
175+
}
176+
b.Raw("VST1.P [V0.S4, V1.S4, V2.S4, V3.S4], 64(R0)").
177+
Raw("SUB $16, R2").Raw("B block").
178+
Raw("tail:").Raw("CBZ R2, done").
179+
Raw("FMOVS.P 4(R1), F0").Raw("FSQRTS F0, F0").Raw("FMOVS.P F0, 4(R0)").
180+
Raw("SUB $1, R2").Raw("B tail").
181+
Raw("done:").Ret()
182+
return b.Func()
183+
}
184+
159185
// bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
160186
// 16 floats per iteration on the .S4 arrangement (VOP Vm, Vn, Vd: Vd = Vn OP
161187
// Vm), then a scalar tail with sop. Each lane is one IEEE float32 operation,

‎internal/kernels/asmgen/loong64/gen.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ func main() {
4242
}
4343
f.Add(gemmKernel())
4444
f.Add(blocks8Kernel())
45+
f.Add(sqrt32Kernel())
4546
if err := os.WriteFile("sum_loong64.s", []byte(f.String()), 0o644); err != nil {
4647
fmt.Fprintln(os.Stderr, err)
4748
os.Exit(1)
@@ -170,6 +171,32 @@ func binKernel(name, vec, sca string) *emit.Function {
170171
return b.Func()
171172
}
172173

174+
// sqrt32Kernel builds sqrt32LSX(dst, src *float32, n int): vfsqrt.s over four
175+
// 128-bit registers, 16 floats per iteration, SQRTF for the tail; the
176+
// correctly rounded float32 square root, bit-identical to
177+
// float32(math.Sqrt(float64(x))).
178+
func sqrt32Kernel() *emit.Function {
179+
sig := loong64.Layout([]string{"dst", "src", "n"}, []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil)
180+
b := loong64.NewFunc("sqrt32LSX", sig, 0)
181+
b.LoadArg("dst", "R4").LoadArg("src", "R6").LoadArg("n", "R5")
182+
b.Label("qloop").Raw("MOVV $16, R12").Raw("BLT R5, R12, qtail")
183+
for k := 0; k < 4; k++ {
184+
b.Raw("VMOVQ %d(R6), V%d", 16*k, k)
185+
}
186+
for k := 0; k < 4; k++ {
187+
b.Raw("VFSQRTF V%d, V%d", k, 4+k)
188+
}
189+
for k := 0; k < 4; k++ {
190+
b.Raw("VMOVQ V%d, %d(R4)", 4+k, 16*k)
191+
}
192+
b.Raw("ADDV $64, R4").Raw("ADDV $64, R6").Raw("ADDV $-16, R5").Raw("JMP qloop")
193+
b.Label("qtail").Raw("BEQ R5, qdone").
194+
Raw("MOVF 0(R6), F1").Raw("SQRTF F1, F1").Raw("MOVF F1, 0(R4)").
195+
Raw("ADDV $4, R4").Raw("ADDV $4, R6").Raw("ADDV $-1, R5").Raw("JMP qtail")
196+
b.Label("qdone").Ret()
197+
return b.Func()
198+
}
199+
173200
// bin32Kernel builds name(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
174201
// 16 floats per iteration over four 128-bit LSX registers (four independent
175202
// chains, as the block kernel's lesson on one LASX register taught), a scalar

‎internal/kernels/asmgen/ppc64le/gen.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ func main() {
4242
}
4343
f.Add(gemmKernel())
4444
f.Add(blocks8Kernel())
45+
f.Add(sqrt32Kernel())
4546
if err := os.WriteFile("sum_ppc64le.s", []byte(f.String()), 0o644); err != nil {
4647
fmt.Fprintln(os.Stderr, err)
4748
os.Exit(1)
@@ -195,6 +196,32 @@ func binKernel(name, op, sca string) *emit.Function {
195196
return b.Func()
196197
}
197198

199+
// sqrt32Kernel builds sqrt32VSX(dst, src *float32, n int): xvsqrtsp over 16
200+
// floats per iteration, FSQRTS for the tail; the correctly rounded float32
201+
// square root, bit-identical to float32(math.Sqrt(float64(x))).
202+
func sqrt32Kernel() *emit.Function {
203+
sig := ppc64.Layout([]string{"dst", "src", "n"}, []ppc64.Type{ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil)
204+
b := ppc64.NewFunc("sqrt32VSX", sig, 0)
205+
b.LoadArg("dst", "R3").LoadArg("src", "R5").LoadArg("n", "R4")
206+
offsets(b, false)
207+
b.Label("qloop").Raw("CMP R4, $16").Raw("BLT qtail")
208+
for k := 0; k < 4; k++ {
209+
b.Raw("LXVW4X (R5)(%s), VS%d", vidx(k), 32+k)
210+
}
211+
for k := 0; k < 4; k++ {
212+
b.XVSQRTSP(36+k, 32+k)
213+
}
214+
for k := 0; k < 4; k++ {
215+
b.Raw("STXVW4X VS%d, (R3)(%s)", 36+k, vidx(k))
216+
}
217+
b.Raw("ADD $64, R3").Raw("ADD $64, R5").Raw("ADD $-16, R4").Raw("BR qloop")
218+
b.Label("qtail").Raw("CMP R4, $0").Raw("BEQ qdone").
219+
Raw("FMOVS 0(R5), F1").Raw("FSQRTS F1, F1").Raw("FMOVS F1, 0(R3)").
220+
Raw("ADD $4, R3").Raw("ADD $4, R5").Raw("ADD $-1, R4").Raw("BR qtail")
221+
b.Label("qdone").Ret()
222+
return b.Func()
223+
}
224+
198225
// bin32Kernel builds name(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
199226
// 16 floats per iteration with the VSX single-precision op, a scalar tail with
200227
// sca. LXVW4X/STXVW4X put each word at the same lane in every operand and back

0 commit comments

Comments
 (0)