Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
complex64/complex128, chosen at run time with **NumPy 2 promotion** (`AsType`,
`ZerosOf`, `FromSlice`, typed access with `Data[T]`/`Item[T]`, weak Go
scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated
one. Float32 has SIMD `Sum` and `Add`/`Sub`/`Mul`/`Div` on every 64-bit
target, bit-identical to its scalar loops
one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every
64-bit target, bit-identical to its scalar loops
([docs/design-dtypes.md](docs/design-dtypes.md)).
- **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/
`Eye`/`Identity`.
Expand Down Expand Up @@ -152,8 +152,8 @@ and `Max` by 92–96% ([docs/perf.md](docs/perf.md), *SIMD coverage*). It is a
> pair Go supports; v0.1.0 and v0.2.0 were also run on real amd64, arm64,
> ppc64le, riscv64 and loong64 hardware, v0.3.0 on real amd64, arm64, ppc64le
> and loong64. See **[docs/plan-ndarray.md](docs/plan-ndarray.md)**
> for the roadmap (float32 SIMD for sqrt, max/min and the products; linear
> algebra decompositions).
> for the roadmap (float32 SIMD for max/min and the products; linear algebra
> decompositions).

## Why this module?

Expand Down
6 changes: 6 additions & 0 deletions SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none.

## Audit log

**2026-10-20 (v0.19.0).** `kernels.Sqrt32` follows `Bin32`'s guards: it
reslices `src` to `len(dst)` before the call (a short `src` panics in Go,
tested), never passes an empty slice to a kernel, and keeps the LASX, V and
VXE gates. The PROT_NONE fence now also covers it, and passes on Zen 3,
Apple M, POWER9, IBM Z, SpacemiT X60 and Loongson 3C5000L.

**2026-10-18 (v0.18.0).** The float32 elementwise kernels (`kernels.Bin32`,
on all six 64-bit targets) take raw pointers and a count. `Bin32` reslices `a`
and `b` to `len(dst)` before the call, so a short operand panics in Go rather
Expand Down
8 changes: 7 additions & 1 deletion arith.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"fmt"
"math/cmplx"
"reflect"

"github.com/go-ndarray/ndarray/internal/kernels"
)

// The operations on dtypes other than Float64. Each public method keeps its
Expand Down Expand Up @@ -224,7 +226,11 @@ func (a *Array) mathUnary(code unaryCode, f64 func(*Array) *Array) *Array {
d := makeStore(dt, a.Size())
switch s := src.(type) {
case []float32:
unaryFloat32(realFuncs[code], d.([]float32), s)
if code == uSqrt {
kernels.Sqrt32(d.([]float32), s) // the same correctly rounded result
} else {
unaryFloat32(realFuncs[code], d.([]float32), s)
}
case []complex64:
unaryComplex(a.complexFunc(code), d.([]complex64), s)
case []complex128:
Expand Down
47 changes: 47 additions & 0 deletions benchmarks/dtype_bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,53 @@ func BenchmarkDTypeAdd(b *testing.B) {
}
}

func BenchmarkDTypeSqrt(b *testing.B) {
for _, dt := range []nd.DType{nd.Float64, nd.Float32} {
for _, n := range []int{1 << 14, 1 << 20} {
x := vec(n).AsType(dt)
b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) {
b.SetBytes(int64(n * dt.Size()))
for i := 0; i < b.N; i++ {
_ = x.Sqrt()
}
})
}
}
}

func BenchmarkDTypeMax(b *testing.B) {
for _, dt := range []nd.DType{nd.Float64, nd.Float32} {
for _, n := range []int{1 << 14, 1 << 20} {
x := vec(n).AsType(dt)
b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) {
b.SetBytes(int64(n * dt.Size()))
for i := 0; i < b.N; i++ {
_, _ = x.Max()
}
})
}
}
}

// BenchmarkDTypeMatMul is a square product, 2·n³ flops per op.
func BenchmarkDTypeMatMul(b *testing.B) {
for _, dt := range []nd.DType{nd.Float64, nd.Float32} {
for _, n := range []int{64, 256} {
x, err := vec(n * n).AsType(dt).Reshape(n, n)
if err != nil {
b.Fatal(err)
}
b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
if _, err := x.MatMul(x); err != nil {
b.Fatal(err)
}
}
})
}
}
}

// BenchmarkDTypeAddInto is Add without the allocation, the analogue of
// np.add(x, y, out=out): it measures the kernel, where BenchmarkDTypeAdd also
// measures allocating and clearing the result.
Expand Down
42 changes: 42 additions & 0 deletions docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -976,6 +976,48 @@ POWER9, one core, against NumPy 2.5.3:
float64 `AddInto` against `np.add(out=)`: 5.70 against 7.75 µs (2¹⁴), and
724 against 768 µs (2²⁰).

### float32 survey against NumPy, and `Sqrt` (2026-10-20)

The rest of the float32 surface, on the same POWER9 core against NumPy 2.5.3:

| | ours | NumPy | |
|---|---:|---:|---|
| float32 `Max`, 2¹⁴ / 2²⁰ | 84.7 µs / 4.86 ms | 7.13 µs / 0.324 ms | **11.9× / 15× slower** |
| float64 `Max`, 2¹⁴ / 2²⁰ | 51.6 µs / 3.33 ms | 11.8 µs / 0.615 ms | 4.4× / 5.4× slower |
| float32 `Sqrt`, 2¹⁴ / 2²⁰ | 100 µs / 6.3 ms | 14.7 µs / 0.835 ms | 6.8× / 7.5× slower |
| float64 `Sqrt`, 2¹⁴ / 2²⁰ | 52.5 µs / 2.7 ms | 37.4 µs / 2.3 ms | 1.4× / 1.2× slower |
| `MatMul` 256², float32 / float64 | 20.4 / 2.25 ms | 30.4 / 29.8 ms | NumPy here has no BLAS |

Read each row against the one below it:

- float32 `Max` has no kernel anywhere, and on ppc64le neither has float64:
`xvmaxdp` does not propagate NaN as NumPy does.
- float32 `Sqrt` called `math.Sqrt` through a function value for every
element.
- float32 `MatMul` is 9× slower than our own float64 `MatMul`: float32 has no
packed GEMM.

`Sqrt` is fixed here. `kernels.Sqrt32` runs `SQRTPS`, NEON `.S4`,
`xvsqrtsp`, `vfsqsb` (z14), LSX `vfsqrt.s` or RVV E32. Each is the correctly
rounded float32 square root, so `float32(math.Sqrt(float64(x)))` bit for bit:
float64 carries more than twice float32's precision.

The kernel tests (special values, every length to 70, in place) and the
PROT_NONE fence pass on Zen 3, Apple M, POWER9, IBM Z, SpacemiT X60 and
Loongson 3C5000L. Measured with fleettools' `abbench` (base and head
alternated, 8 rounds, pinned core, `--max-busy 15`):

| float32 `Sqrt`, POWER9 | before | after | NumPy |
|---|---:|---:|---:|
| 2¹⁴ | 102.9 µs | **23.4 µs** (−77%) | 14.7 µs |
| 2²⁰ | 6.21 ms | **1.08 ms** (−83%) | 0.835 ms |

The controls (float64 `Sqrt`, `Add`, `AddInto`) are flat. The gap left to
NumPy is the allocation, the open defect below.

Still to do, in this order: `Max`/`Min` (float32 everywhere, float64 on
ppc64le, loong64 and riscv64), then a float32 GEMM.

**Open defect: the allocating operations.** `Add` is still 1.5–3.2× slower
than NumPy's `a+b`, for float64 as well (21.9 against 8.48 µs at 2¹⁴). The
kernel is no longer the difference, as `AddInto` shows; the allocation is.
Expand Down
6 changes: 4 additions & 2 deletions docs/plan-ndarray.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,10 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`).
with NumPy 2.x promotion; see [design-dtypes.md](design-dtypes.md). The
`fft` bridge shipped in v0.10.0 (package `fft`, over go-fft). float32 has
SIMD kernels on all six 64-bit targets for `Sum` (v0.14.0–v0.17.0) and for
`Add`/`Sub`/`Mul`/`Div` (v0.18.0). Still pure Go for float32: `Sqrt`,
`Max`/`Min`, `Dot`/`MatMul` and the transcendental ufuncs.
`Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go
for float32: `Max`/`Min` (11.9–15× slower than NumPy on a POWER9),
`Dot`/`MatMul` (9× slower than our float64 GEMM) and the transcendental
ufuncs.
- **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic,
comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`,
`Where` and the per-axis reductions, in every dtype. Not done: a public
Expand Down
32 changes: 32 additions & 0 deletions internal/kernels/asmgen/amd64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func main() {
f.Add(bin32Kernel("sub32SSE2", "SUBPS", "SUBSS"))
f.Add(bin32Kernel("mul32SSE2", "MULPS", "MULSS"))
f.Add(bin32Kernel("div32SSE2", "DIVPS", "DIVSS"))
f.Add(sqrt32Kernel())

if err := os.WriteFile("sum_amd64.s", []byte(f.String()), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
Expand Down Expand Up @@ -329,6 +330,37 @@ func binKernel(name, vec, sca string) *emit.Function {
return b.Func()
}

// sqrt32Kernel builds sqrt32SSE2(dst, src *float32, n int): SQRTPS over 16
// floats per iteration, SQRTSS for the tail. Both are the correctly rounded
// IEEE float32 square root, which is float32(math.Sqrt(float64(x))) bit for
// bit (float64 carries more than twice float32's precision).
func sqrt32Kernel() *emit.Function {
sig := amd64.Layout(
[]string{"dst", "src", "n"},
[]amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64},
nil, nil,
)
b := amd64.NewFunc("sqrt32SSE2", sig, 0)
b.LoadArg("dst", "DI").LoadArg("src", "SI").LoadArg("n", "CX")
b.Raw("qblock:").Raw("CMPQ CX, $16").Raw("JL qtail")
for r := 0; r < 4; r++ {
b.Raw("MOVUPS %d(SI), X%d", 16*r, r)
}
for r := 0; r < 4; r++ {
b.Raw("SQRTPS X%d, X%d", r, r)
}
for r := 0; r < 4; r++ {
b.Raw("MOVUPS X%d, %d(DI)", r, 16*r)
}
b.Raw("ADDQ $64, SI").Raw("ADDQ $64, DI").Raw("SUBQ $16, CX").Raw("JMP qblock").
Raw("qtail:").Raw("TESTQ CX, CX").Raw("JZ qdone").
Raw("MOVSS (SI), X0").Raw("SQRTSS X0, X0").Raw("MOVSS X0, (DI)").
Raw("ADDQ $4, SI").Raw("ADDQ $4, DI").Raw("DECQ CX").Raw("JMP qtail").
Raw("qdone:")
b.Ret()
return b.Func()
}

// bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
// 16 floats per iteration over four XMM registers with the packed op, then a
// scalar tail with sca. Each lane is one IEEE float32 operation, so the result
Expand Down
26 changes: 26 additions & 0 deletions internal/kernels/asmgen/arm64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func main() {
f.Add(bin32Kernel("sub32NEON", "VFSUB", "FSUBS"))
f.Add(bin32Kernel("mul32NEON", "VFMUL", "FMULS"))
f.Add(bin32Kernel("div32NEON", "VFDIV", "FDIVS"))
f.Add(sqrt32Kernel())
f.Add(extremumKernel("maxNEON", "VFMAX", "FMAXD"))
f.Add(extremumKernel("minNEON", "VFMIN", "FMIND"))
f.Add(gemmKernel())
Expand Down Expand Up @@ -156,6 +157,31 @@ func binKernel(name, vop, sop string) *emit.Function {
return b.Func()
}

// sqrt32Kernel builds sqrt32NEON(dst, src *float32, n int): VFSQRT on .S4
// over 16 floats per iteration, FSQRTS for the tail; the correctly rounded
// float32 square root, bit-identical to float32(math.Sqrt(float64(x))).
func sqrt32Kernel() *emit.Function {
sig := arm64.Layout(
[]string{"dst", "src", "n"},
[]arm64.Type{arm64.Ptr, arm64.Ptr, arm64.Int64},
nil, nil,
)
b := arm64.NewFunc("sqrt32NEON", sig, 0)
b.LoadArg("dst", "R0").LoadArg("src", "R1").LoadArg("n", "R2").
Raw("block:").Raw("CMP $16, R2").Raw("BLT tail").
Raw("VLD1.P 64(R1), [V0.S4, V1.S4, V2.S4, V3.S4]")
for r := 0; r < 4; r++ {
b.Raw(fmt.Sprintf("VFSQRT V%d.S4, V%d.S4", r, r))
}
b.Raw("VST1.P [V0.S4, V1.S4, V2.S4, V3.S4], 64(R0)").
Raw("SUB $16, R2").Raw("B block").
Raw("tail:").Raw("CBZ R2, done").
Raw("FMOVS.P 4(R1), F0").Raw("FSQRTS F0, F0").Raw("FMOVS.P F0, 4(R0)").
Raw("SUB $1, R2").Raw("B tail").
Raw("done:").Ret()
return b.Func()
}

// bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
// 16 floats per iteration on the .S4 arrangement (VOP Vm, Vn, Vd: Vd = Vn OP
// Vm), then a scalar tail with sop. Each lane is one IEEE float32 operation,
Expand Down
27 changes: 27 additions & 0 deletions internal/kernels/asmgen/loong64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func main() {
}
f.Add(gemmKernel())
f.Add(blocks8Kernel())
f.Add(sqrt32Kernel())
if err := os.WriteFile("sum_loong64.s", []byte(f.String()), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
Expand Down Expand Up @@ -170,6 +171,32 @@ func binKernel(name, vec, sca string) *emit.Function {
return b.Func()
}

// sqrt32Kernel builds sqrt32LSX(dst, src *float32, n int): vfsqrt.s over four
// 128-bit registers, 16 floats per iteration, SQRTF for the tail; the
// correctly rounded float32 square root, bit-identical to
// float32(math.Sqrt(float64(x))).
func sqrt32Kernel() *emit.Function {
sig := loong64.Layout([]string{"dst", "src", "n"}, []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil)
b := loong64.NewFunc("sqrt32LSX", sig, 0)
b.LoadArg("dst", "R4").LoadArg("src", "R6").LoadArg("n", "R5")
b.Label("qloop").Raw("MOVV $16, R12").Raw("BLT R5, R12, qtail")
for k := 0; k < 4; k++ {
b.Raw("VMOVQ %d(R6), V%d", 16*k, k)
}
for k := 0; k < 4; k++ {
b.Raw("VFSQRTF V%d, V%d", k, 4+k)
}
for k := 0; k < 4; k++ {
b.Raw("VMOVQ V%d, %d(R4)", 4+k, 16*k)
}
b.Raw("ADDV $64, R4").Raw("ADDV $64, R6").Raw("ADDV $-16, R5").Raw("JMP qloop")
b.Label("qtail").Raw("BEQ R5, qdone").
Raw("MOVF 0(R6), F1").Raw("SQRTF F1, F1").Raw("MOVF F1, 0(R4)").
Raw("ADDV $4, R4").Raw("ADDV $4, R6").Raw("ADDV $-1, R5").Raw("JMP qtail")
b.Label("qdone").Ret()
return b.Func()
}

// bin32Kernel builds name(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
// 16 floats per iteration over four 128-bit LSX registers (four independent
// chains, as the block kernel's lesson on one LASX register taught), a scalar
Expand Down
27 changes: 27 additions & 0 deletions internal/kernels/asmgen/ppc64le/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func main() {
}
f.Add(gemmKernel())
f.Add(blocks8Kernel())
f.Add(sqrt32Kernel())
if err := os.WriteFile("sum_ppc64le.s", []byte(f.String()), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
Expand Down Expand Up @@ -195,6 +196,32 @@ func binKernel(name, op, sca string) *emit.Function {
return b.Func()
}

// sqrt32Kernel builds sqrt32VSX(dst, src *float32, n int): xvsqrtsp over 16
// floats per iteration, FSQRTS for the tail; the correctly rounded float32
// square root, bit-identical to float32(math.Sqrt(float64(x))).
func sqrt32Kernel() *emit.Function {
sig := ppc64.Layout([]string{"dst", "src", "n"}, []ppc64.Type{ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil)
b := ppc64.NewFunc("sqrt32VSX", sig, 0)
b.LoadArg("dst", "R3").LoadArg("src", "R5").LoadArg("n", "R4")
offsets(b, false)
b.Label("qloop").Raw("CMP R4, $16").Raw("BLT qtail")
for k := 0; k < 4; k++ {
b.Raw("LXVW4X (R5)(%s), VS%d", vidx(k), 32+k)
}
for k := 0; k < 4; k++ {
b.XVSQRTSP(36+k, 32+k)
}
for k := 0; k < 4; k++ {
b.Raw("STXVW4X VS%d, (R3)(%s)", 36+k, vidx(k))
}
b.Raw("ADD $64, R3").Raw("ADD $64, R5").Raw("ADD $-16, R4").Raw("BR qloop")
b.Label("qtail").Raw("CMP R4, $0").Raw("BEQ qdone").
Raw("FMOVS 0(R5), F1").Raw("FSQRTS F1, F1").Raw("FMOVS F1, 0(R3)").
Raw("ADD $4, R3").Raw("ADD $4, R5").Raw("ADD $-1, R4").Raw("BR qtail")
b.Label("qdone").Ret()
return b.Func()
}

// bin32Kernel builds name(dst, a, b *float32, n int): dst[i] = a[i] OP b[i],
// 16 floats per iteration with the VSX single-precision op, a scalar tail with
// sca. LXVW4X/STXVW4X put each word at the same lane in every operand and back
Expand Down
22 changes: 22 additions & 0 deletions internal/kernels/asmgen/riscv64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func main() {
}
f.Add(gemmKernel())
f.Add(blocks8Kernel())
f.Add(sqrt32Kernel())
if err := os.WriteFile("sum_riscv64.s", []byte(f.String()), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
Expand Down Expand Up @@ -129,6 +130,27 @@ func binKernel(name, op string) *emit.Function {
return b.Func()
}

// sqrt32Kernel builds sqrt32RVV(dst, src *float32, n int) for n >= 1:
// vfsqrt.v on 32-bit elements, strip-mined; the correctly rounded float32
// square root, bit-identical to float32(math.Sqrt(float64(x))).
func sqrt32Kernel() *emit.Function {
sig := riscv64.Layout([]string{"dst", "src", "n"},
[]riscv64.Type{riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil)
b := riscv64.NewFunc("sqrt32RVV", sig, 0)
b.LoadArg("dst", "X10").LoadArg("src", "X11").LoadArg("n", "X13")
b.Raw("loop:").
Raw("VSETVLI X13, E32, M8, TA, MA, X14").
Raw("VLE32V (X11), V0").
Raw("VFSQRTV V0, V16").
Raw("VSE32V V16, (X10)").
Raw("SLLI $2, X14, X17") // vl elements of 4 bytes
for _, p := range []string{"X10", "X11"} {
b.Raw("ADD X17, %s, %s", p, p)
}
b.Raw("SUB X14, X13, X13").Raw("BNEZ X13, loop").Ret()
return b.Func()
}

// bin32Kernel builds NAME(dst, a, b *float32, n int) for n >= 1: dst = a OP b
// on 32-bit elements, strip-mined by VSETVLI (no tail loop). Each lane is one
// IEEE float32 operation, so the result is the scalar loop's bit for bit.
Expand Down
Loading
Loading