diff --git a/README.md b/README.md index 60ee7e3..d3acbee 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ complex64/complex128, chosen at run time with **NumPy 2 promotion** (`AsType`, `ZerosOf`, `FromSlice`, typed access with `Data[T]`/`Item[T]`, weak Go scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated - one. Float32 has SIMD `Sum` and `Add`/`Sub`/`Mul`/`Div` on every 64-bit - target, bit-identical to its scalar loops + one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every + 64-bit target, bit-identical to its scalar loops ([docs/design-dtypes.md](docs/design-dtypes.md)). - **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/ `Eye`/`Identity`. @@ -152,8 +152,8 @@ and `Max` by 92–96% ([docs/perf.md](docs/perf.md), *SIMD coverage*). It is a > pair Go supports; v0.1.0 and v0.2.0 were also run on real amd64, arm64, > ppc64le, riscv64 and loong64 hardware, v0.3.0 on real amd64, arm64, ppc64le > and loong64. See **[docs/plan-ndarray.md](docs/plan-ndarray.md)** -> for the roadmap (float32 SIMD for sqrt, max/min and the products; linear -> algebra decompositions). +> for the roadmap (float32 SIMD for max/min and the products; linear algebra +> decompositions). ## Why this module? diff --git a/SECURITY.md b/SECURITY.md index 477f677..ca8e41b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,12 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-20 (v0.19.0).** `kernels.Sqrt32` follows `Bin32`'s guards: it +reslices `src` to `len(dst)` before the call (a short `src` panics in Go, +tested), never passes an empty slice to a kernel, and keeps the LASX, V and +VXE gates. The PROT_NONE fence now also covers it, and passes on Zen 3, +Apple M, POWER9, IBM Z, SpacemiT X60 and Loongson 3C5000L. + **2026-10-18 (v0.18.0).** The float32 elementwise kernels (`kernels.Bin32`, on all six 64-bit targets) take raw pointers and a count. `Bin32` reslices `a` and `b` to `len(dst)` before the call, so a short operand panics in Go rather diff --git a/arith.go b/arith.go index d3d8225..43770e5 100644 --- a/arith.go +++ b/arith.go @@ -4,6 +4,8 @@ import ( "fmt" "math/cmplx" "reflect" + + "github.com/go-ndarray/ndarray/internal/kernels" ) // The operations on dtypes other than Float64. Each public method keeps its @@ -224,7 +226,11 @@ func (a *Array) mathUnary(code unaryCode, f64 func(*Array) *Array) *Array { d := makeStore(dt, a.Size()) switch s := src.(type) { case []float32: - unaryFloat32(realFuncs[code], d.([]float32), s) + if code == uSqrt { + kernels.Sqrt32(d.([]float32), s) // the same correctly rounded result + } else { + unaryFloat32(realFuncs[code], d.([]float32), s) + } case []complex64: unaryComplex(a.complexFunc(code), d.([]complex64), s) case []complex128: diff --git a/benchmarks/dtype_bench_test.go b/benchmarks/dtype_bench_test.go index c7629e0..c431e11 100644 --- a/benchmarks/dtype_bench_test.go +++ b/benchmarks/dtype_bench_test.go @@ -42,6 +42,53 @@ func BenchmarkDTypeAdd(b *testing.B) { } } +func BenchmarkDTypeSqrt(b *testing.B) { + for _, dt := range []nd.DType{nd.Float64, nd.Float32} { + for _, n := range []int{1 << 14, 1 << 20} { + x := vec(n).AsType(dt) + b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) { + b.SetBytes(int64(n * dt.Size())) + for i := 0; i < b.N; i++ { + _ = x.Sqrt() + } + }) + } + } +} + +func BenchmarkDTypeMax(b *testing.B) { + for _, dt := range []nd.DType{nd.Float64, nd.Float32} { + for _, n := range []int{1 << 14, 1 << 20} { + x := vec(n).AsType(dt) + b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) { + b.SetBytes(int64(n * dt.Size())) + for i := 0; i < b.N; i++ { + _, _ = x.Max() + } + }) + } + } +} + +// BenchmarkDTypeMatMul is a square product, 2·n³ flops per op. +func BenchmarkDTypeMatMul(b *testing.B) { + for _, dt := range []nd.DType{nd.Float64, nd.Float32} { + for _, n := range []int{64, 256} { + x, err := vec(n * n).AsType(dt).Reshape(n, n) + if err != nil { + b.Fatal(err) + } + b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := x.MatMul(x); err != nil { + b.Fatal(err) + } + } + }) + } + } +} + // BenchmarkDTypeAddInto is Add without the allocation, the analogue of // np.add(x, y, out=out): it measures the kernel, where BenchmarkDTypeAdd also // measures allocating and clearing the result. diff --git a/docs/perf.md b/docs/perf.md index 7354a6d..2694ef8 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -976,6 +976,48 @@ POWER9, one core, against NumPy 2.5.3: float64 `AddInto` against `np.add(out=)`: 5.70 against 7.75 µs (2¹⁴), and 724 against 768 µs (2²⁰). +### float32 survey against NumPy, and `Sqrt` (2026-10-20) + +The rest of the float32 surface, on the same POWER9 core against NumPy 2.5.3: + +| | ours | NumPy | | +|---|---:|---:|---| +| float32 `Max`, 2¹⁴ / 2²⁰ | 84.7 µs / 4.86 ms | 7.13 µs / 0.324 ms | **11.9× / 15× slower** | +| float64 `Max`, 2¹⁴ / 2²⁰ | 51.6 µs / 3.33 ms | 11.8 µs / 0.615 ms | 4.4× / 5.4× slower | +| float32 `Sqrt`, 2¹⁴ / 2²⁰ | 100 µs / 6.3 ms | 14.7 µs / 0.835 ms | 6.8× / 7.5× slower | +| float64 `Sqrt`, 2¹⁴ / 2²⁰ | 52.5 µs / 2.7 ms | 37.4 µs / 2.3 ms | 1.4× / 1.2× slower | +| `MatMul` 256², float32 / float64 | 20.4 / 2.25 ms | 30.4 / 29.8 ms | NumPy here has no BLAS | + +Read each row against the one below it: + +- float32 `Max` has no kernel anywhere, and on ppc64le neither has float64: + `xvmaxdp` does not propagate NaN as NumPy does. +- float32 `Sqrt` called `math.Sqrt` through a function value for every + element. +- float32 `MatMul` is 9× slower than our own float64 `MatMul`: float32 has no + packed GEMM. + +`Sqrt` is fixed here. `kernels.Sqrt32` runs `SQRTPS`, NEON `.S4`, +`xvsqrtsp`, `vfsqsb` (z14), LSX `vfsqrt.s` or RVV E32. Each is the correctly +rounded float32 square root, so `float32(math.Sqrt(float64(x)))` bit for bit: +float64 carries more than twice float32's precision. + +The kernel tests (special values, every length to 70, in place) and the +PROT_NONE fence pass on Zen 3, Apple M, POWER9, IBM Z, SpacemiT X60 and +Loongson 3C5000L. Measured with fleettools' `abbench` (base and head +alternated, 8 rounds, pinned core, `--max-busy 15`): + +| float32 `Sqrt`, POWER9 | before | after | NumPy | +|---|---:|---:|---:| +| 2¹⁴ | 102.9 µs | **23.4 µs** (−77%) | 14.7 µs | +| 2²⁰ | 6.21 ms | **1.08 ms** (−83%) | 0.835 ms | + +The controls (float64 `Sqrt`, `Add`, `AddInto`) are flat. The gap left to +NumPy is the allocation, the open defect below. + +Still to do, in this order: `Max`/`Min` (float32 everywhere, float64 on +ppc64le, loong64 and riscv64), then a float32 GEMM. + **Open defect: the allocating operations.** `Add` is still 1.5–3.2× slower than NumPy's `a+b`, for float64 as well (21.9 against 8.48 µs at 2¹⁴). The kernel is no longer the difference, as `AddInto` shows; the allocation is. diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index 3aec3ba..8df0b08 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -156,8 +156,10 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). with NumPy 2.x promotion; see [design-dtypes.md](design-dtypes.md). The `fft` bridge shipped in v0.10.0 (package `fft`, over go-fft). float32 has SIMD kernels on all six 64-bit targets for `Sum` (v0.14.0–v0.17.0) and for - `Add`/`Sub`/`Mul`/`Div` (v0.18.0). Still pure Go for float32: `Sqrt`, - `Max`/`Min`, `Dot`/`MatMul` and the transcendental ufuncs. + `Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go + for float32: `Max`/`Min` (11.9–15× slower than NumPy on a POWER9), + `Dot`/`MatMul` (9× slower than our float64 GEMM) and the transcendental + ufuncs. - **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic, comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`, `Where` and the per-axis reductions, in every dtype. Not done: a public diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index c1ebc62..507541a 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -54,6 +54,7 @@ func main() { f.Add(bin32Kernel("sub32SSE2", "SUBPS", "SUBSS")) f.Add(bin32Kernel("mul32SSE2", "MULPS", "MULSS")) f.Add(bin32Kernel("div32SSE2", "DIVPS", "DIVSS")) + f.Add(sqrt32Kernel()) if err := os.WriteFile("sum_amd64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) @@ -329,6 +330,37 @@ func binKernel(name, vec, sca string) *emit.Function { return b.Func() } +// sqrt32Kernel builds sqrt32SSE2(dst, src *float32, n int): SQRTPS over 16 +// floats per iteration, SQRTSS for the tail. Both are the correctly rounded +// IEEE float32 square root, which is float32(math.Sqrt(float64(x))) bit for +// bit (float64 carries more than twice float32's precision). +func sqrt32Kernel() *emit.Function { + sig := amd64.Layout( + []string{"dst", "src", "n"}, + []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64}, + nil, nil, + ) + b := amd64.NewFunc("sqrt32SSE2", sig, 0) + b.LoadArg("dst", "DI").LoadArg("src", "SI").LoadArg("n", "CX") + b.Raw("qblock:").Raw("CMPQ CX, $16").Raw("JL qtail") + for r := 0; r < 4; r++ { + b.Raw("MOVUPS %d(SI), X%d", 16*r, r) + } + for r := 0; r < 4; r++ { + b.Raw("SQRTPS X%d, X%d", r, r) + } + for r := 0; r < 4; r++ { + b.Raw("MOVUPS X%d, %d(DI)", r, 16*r) + } + b.Raw("ADDQ $64, SI").Raw("ADDQ $64, DI").Raw("SUBQ $16, CX").Raw("JMP qblock"). + Raw("qtail:").Raw("TESTQ CX, CX").Raw("JZ qdone"). + Raw("MOVSS (SI), X0").Raw("SQRTSS X0, X0").Raw("MOVSS X0, (DI)"). + Raw("ADDQ $4, SI").Raw("ADDQ $4, DI").Raw("DECQ CX").Raw("JMP qtail"). + Raw("qdone:") + b.Ret() + return b.Func() +} + // bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i], // 16 floats per iteration over four XMM registers with the packed op, then a // scalar tail with sca. Each lane is one IEEE float32 operation, so the result diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index 059d364..a1f9d10 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -51,6 +51,7 @@ func main() { f.Add(bin32Kernel("sub32NEON", "VFSUB", "FSUBS")) f.Add(bin32Kernel("mul32NEON", "VFMUL", "FMULS")) f.Add(bin32Kernel("div32NEON", "VFDIV", "FDIVS")) + f.Add(sqrt32Kernel()) f.Add(extremumKernel("maxNEON", "VFMAX", "FMAXD")) f.Add(extremumKernel("minNEON", "VFMIN", "FMIND")) f.Add(gemmKernel()) @@ -156,6 +157,31 @@ func binKernel(name, vop, sop string) *emit.Function { return b.Func() } +// sqrt32Kernel builds sqrt32NEON(dst, src *float32, n int): VFSQRT on .S4 +// over 16 floats per iteration, FSQRTS for the tail; the correctly rounded +// float32 square root, bit-identical to float32(math.Sqrt(float64(x))). +func sqrt32Kernel() *emit.Function { + sig := arm64.Layout( + []string{"dst", "src", "n"}, + []arm64.Type{arm64.Ptr, arm64.Ptr, arm64.Int64}, + nil, nil, + ) + b := arm64.NewFunc("sqrt32NEON", sig, 0) + b.LoadArg("dst", "R0").LoadArg("src", "R1").LoadArg("n", "R2"). + Raw("block:").Raw("CMP $16, R2").Raw("BLT tail"). + Raw("VLD1.P 64(R1), [V0.S4, V1.S4, V2.S4, V3.S4]") + for r := 0; r < 4; r++ { + b.Raw(fmt.Sprintf("VFSQRT V%d.S4, V%d.S4", r, r)) + } + b.Raw("VST1.P [V0.S4, V1.S4, V2.S4, V3.S4], 64(R0)"). + Raw("SUB $16, R2").Raw("B block"). + Raw("tail:").Raw("CBZ R2, done"). + Raw("FMOVS.P 4(R1), F0").Raw("FSQRTS F0, F0").Raw("FMOVS.P F0, 4(R0)"). + Raw("SUB $1, R2").Raw("B tail"). + Raw("done:").Ret() + return b.Func() +} + // bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i], // 16 floats per iteration on the .S4 arrangement (VOP Vm, Vn, Vd: Vd = Vn OP // Vm), then a scalar tail with sop. Each lane is one IEEE float32 operation, diff --git a/internal/kernels/asmgen/loong64/gen.go b/internal/kernels/asmgen/loong64/gen.go index c37003a..0dd17d0 100644 --- a/internal/kernels/asmgen/loong64/gen.go +++ b/internal/kernels/asmgen/loong64/gen.go @@ -42,6 +42,7 @@ func main() { } f.Add(gemmKernel()) f.Add(blocks8Kernel()) + f.Add(sqrt32Kernel()) if err := os.WriteFile("sum_loong64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -170,6 +171,32 @@ func binKernel(name, vec, sca string) *emit.Function { return b.Func() } +// sqrt32Kernel builds sqrt32LSX(dst, src *float32, n int): vfsqrt.s over four +// 128-bit registers, 16 floats per iteration, SQRTF for the tail; the +// correctly rounded float32 square root, bit-identical to +// float32(math.Sqrt(float64(x))). +func sqrt32Kernel() *emit.Function { + sig := loong64.Layout([]string{"dst", "src", "n"}, []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil) + b := loong64.NewFunc("sqrt32LSX", sig, 0) + b.LoadArg("dst", "R4").LoadArg("src", "R6").LoadArg("n", "R5") + b.Label("qloop").Raw("MOVV $16, R12").Raw("BLT R5, R12, qtail") + for k := 0; k < 4; k++ { + b.Raw("VMOVQ %d(R6), V%d", 16*k, k) + } + for k := 0; k < 4; k++ { + b.Raw("VFSQRTF V%d, V%d", k, 4+k) + } + for k := 0; k < 4; k++ { + b.Raw("VMOVQ V%d, %d(R4)", 4+k, 16*k) + } + b.Raw("ADDV $64, R4").Raw("ADDV $64, R6").Raw("ADDV $-16, R5").Raw("JMP qloop") + b.Label("qtail").Raw("BEQ R5, qdone"). + Raw("MOVF 0(R6), F1").Raw("SQRTF F1, F1").Raw("MOVF F1, 0(R4)"). + Raw("ADDV $4, R4").Raw("ADDV $4, R6").Raw("ADDV $-1, R5").Raw("JMP qtail") + b.Label("qdone").Ret() + return b.Func() +} + // bin32Kernel builds name(dst, a, b *float32, n int): dst[i] = a[i] OP b[i], // 16 floats per iteration over four 128-bit LSX registers (four independent // chains, as the block kernel's lesson on one LASX register taught), a scalar diff --git a/internal/kernels/asmgen/ppc64le/gen.go b/internal/kernels/asmgen/ppc64le/gen.go index e7b241b..cec69a0 100644 --- a/internal/kernels/asmgen/ppc64le/gen.go +++ b/internal/kernels/asmgen/ppc64le/gen.go @@ -42,6 +42,7 @@ func main() { } f.Add(gemmKernel()) f.Add(blocks8Kernel()) + f.Add(sqrt32Kernel()) if err := os.WriteFile("sum_ppc64le.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -195,6 +196,32 @@ func binKernel(name, op, sca string) *emit.Function { return b.Func() } +// sqrt32Kernel builds sqrt32VSX(dst, src *float32, n int): xvsqrtsp over 16 +// floats per iteration, FSQRTS for the tail; the correctly rounded float32 +// square root, bit-identical to float32(math.Sqrt(float64(x))). +func sqrt32Kernel() *emit.Function { + sig := ppc64.Layout([]string{"dst", "src", "n"}, []ppc64.Type{ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) + b := ppc64.NewFunc("sqrt32VSX", sig, 0) + b.LoadArg("dst", "R3").LoadArg("src", "R5").LoadArg("n", "R4") + offsets(b, false) + b.Label("qloop").Raw("CMP R4, $16").Raw("BLT qtail") + for k := 0; k < 4; k++ { + b.Raw("LXVW4X (R5)(%s), VS%d", vidx(k), 32+k) + } + for k := 0; k < 4; k++ { + b.XVSQRTSP(36+k, 32+k) + } + for k := 0; k < 4; k++ { + b.Raw("STXVW4X VS%d, (R3)(%s)", 36+k, vidx(k)) + } + b.Raw("ADD $64, R3").Raw("ADD $64, R5").Raw("ADD $-16, R4").Raw("BR qloop") + b.Label("qtail").Raw("CMP R4, $0").Raw("BEQ qdone"). + Raw("FMOVS 0(R5), F1").Raw("FSQRTS F1, F1").Raw("FMOVS F1, 0(R3)"). + Raw("ADD $4, R3").Raw("ADD $4, R5").Raw("ADD $-1, R4").Raw("BR qtail") + b.Label("qdone").Ret() + return b.Func() +} + // bin32Kernel builds name(dst, a, b *float32, n int): dst[i] = a[i] OP b[i], // 16 floats per iteration with the VSX single-precision op, a scalar tail with // sca. LXVW4X/STXVW4X put each word at the same lane in every operand and back diff --git a/internal/kernels/asmgen/riscv64/gen.go b/internal/kernels/asmgen/riscv64/gen.go index d631abb..b859dd5 100644 --- a/internal/kernels/asmgen/riscv64/gen.go +++ b/internal/kernels/asmgen/riscv64/gen.go @@ -38,6 +38,7 @@ func main() { } f.Add(gemmKernel()) f.Add(blocks8Kernel()) + f.Add(sqrt32Kernel()) if err := os.WriteFile("sum_riscv64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -129,6 +130,27 @@ func binKernel(name, op string) *emit.Function { return b.Func() } +// sqrt32Kernel builds sqrt32RVV(dst, src *float32, n int) for n >= 1: +// vfsqrt.v on 32-bit elements, strip-mined; the correctly rounded float32 +// square root, bit-identical to float32(math.Sqrt(float64(x))). +func sqrt32Kernel() *emit.Function { + sig := riscv64.Layout([]string{"dst", "src", "n"}, + []riscv64.Type{riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) + b := riscv64.NewFunc("sqrt32RVV", sig, 0) + b.LoadArg("dst", "X10").LoadArg("src", "X11").LoadArg("n", "X13") + b.Raw("loop:"). + Raw("VSETVLI X13, E32, M8, TA, MA, X14"). + Raw("VLE32V (X11), V0"). + Raw("VFSQRTV V0, V16"). + Raw("VSE32V V16, (X10)"). + Raw("SLLI $2, X14, X17") // vl elements of 4 bytes + for _, p := range []string{"X10", "X11"} { + b.Raw("ADD X17, %s, %s", p, p) + } + b.Raw("SUB X14, X13, X13").Raw("BNEZ X13, loop").Ret() + return b.Func() +} + // bin32Kernel builds NAME(dst, a, b *float32, n int) for n >= 1: dst = a OP b // on 32-bit elements, strip-mined by VSETVLI (no tail loop). Each lane is one // IEEE float32 operation, so the result is the scalar loop's bit for bit. diff --git a/internal/kernels/asmgen/s390x/gen.go b/internal/kernels/asmgen/s390x/gen.go index fb68d03..75e55ef 100644 --- a/internal/kernels/asmgen/s390x/gen.go +++ b/internal/kernels/asmgen/s390x/gen.go @@ -41,6 +41,7 @@ func main() { f.Add(extremumKernel("maxVXE", "VFMAXDB", "WFMAXDB")) f.Add(extremumKernel("minVXE", "VFMINDB", "WFMINDB")) f.Add(blocks8Kernel()) + f.Add(sqrt32Kernel()) if err := os.WriteFile("sum_s390x.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -144,6 +145,29 @@ func binKernel(name, vec, sca string) *emit.Function { return b.Func() } +// sqrt32Kernel builds sqrt32VXE(dst, src *float32, n int): vfsqsb (z14, +// WORD+BYTE+BYTE from go-asmgen) over 16 floats per iteration, FSQRTS for the +// tail; the correctly rounded float32 square root, bit-identical to +// float32(math.Sqrt(float64(x))). +func sqrt32Kernel() *emit.Function { + sig := s390x.Layout([]string{"dst", "src", "n"}, + []s390x.Type{s390x.Ptr, s390x.Ptr, s390x.Int64}, nil, nil) + b := s390x.NewFunc("sqrt32VXE", sig, 0) + b.LoadArg("dst", "R1").LoadArg("src", "R2").LoadArg("n", "R4") + b.Raw("loop:").Raw("CMP R4, $16").Raw("BLT tail"). + Raw("VLM (R2), V0, V3") + for r := 0; r < 4; r++ { + b.VFSQSB(r, r) + } + b.Raw("VSTM V0, V3, (R1)"). + Raw("ADD $64, R1").Raw("ADD $64, R2").Raw("SUB $16, R4").Raw("BR loop"). + Raw("tail:").Raw("CMP R4, $0").Raw("BEQ done"). + Raw("FMOVS (R2), F0").Raw("FSQRTS F0, F0").Raw("FMOVS F0, (R1)"). + Raw("ADD $4, R1").Raw("ADD $4, R2").Raw("SUB $1, R4").Raw("BR tail"). + Raw("done:").Ret() + return b.Func() +} + // bin32Kernel builds NAME(dst, a, b *float32, n int): dst = a OP b, 16 floats // per iteration over four vector registers with the vector-enhancements // facility 1 single-precision op (z14; go-asmgen emits it as WORD+BYTE+BYTE, diff --git a/internal/kernels/bin32.go b/internal/kernels/bin32.go index b7d2dd8..c95618c 100644 --- a/internal/kernels/bin32.go +++ b/internal/kernels/bin32.go @@ -1,5 +1,7 @@ package kernels +import "math" + // The float32 elementwise operations, in the order of the parent package's // binCode: Bin32 takes these codes. const ( @@ -47,6 +49,28 @@ func bin32Go(op int, dst, a, b []float32) { } } +// Sqrt32 writes the square root of src[i] into dst[i] for every element. src +// must be at least as long as dst. A vector kernel runs where the architecture +// has one; the correctly rounded float32 square root is +// float32(math.Sqrt(float64(x))) bit for bit, since float64 carries more than +// twice float32's precision. +func Sqrt32(dst, src []float32) { + n := len(dst) + if n == 0 { + return + } + sqrt32(dst, src[:n]) // the kernel reads n of src +} + +// sqrt32Go is the scalar oracle for Sqrt32, and the implementation where +// there is no kernel. +func sqrt32Go(dst, src []float32) { + src = src[:len(dst)] + for i := range dst { + dst[i] = float32(math.Sqrt(float64(src[i]))) + } +} + // bin32Func is the shape of the generated float32 kernels: n >= 1 elements. type bin32Func func(dst, a, b *float32, n int) diff --git a/internal/kernels/bin32_amd64.go b/internal/kernels/bin32_amd64.go index f4a75ea..25f1d5a 100644 --- a/internal/kernels/bin32_amd64.go +++ b/internal/kernels/bin32_amd64.go @@ -7,6 +7,13 @@ func bin32(op int, dst, a, b []float32) { run32([...]bin32Func{add32SSE2, sub32SSE2, mul32SSE2, div32SSE2}[op], dst, a, b) } +// sqrt32 runs SQRTPS (sum_amd64.s). dst and src have the same length, at +// least 1. +func sqrt32(dst, src []float32) { sqrt32SSE2(&dst[0], &src[0], len(dst)) } + +//go:noescape +func sqrt32SSE2(dst, src *float32, n int) + //go:noescape func add32SSE2(dst, a, b *float32, n int) diff --git a/internal/kernels/bin32_arm64.go b/internal/kernels/bin32_arm64.go index e26ac46..8c68b9a 100644 --- a/internal/kernels/bin32_arm64.go +++ b/internal/kernels/bin32_arm64.go @@ -7,6 +7,13 @@ func bin32(op int, dst, a, b []float32) { run32([...]bin32Func{add32NEON, sub32NEON, mul32NEON, div32NEON}[op], dst, a, b) } +// sqrt32 runs VFSQRT on .S4 (sum_arm64.s). dst and src have the same length, +// at least 1. +func sqrt32(dst, src []float32) { sqrt32NEON(&dst[0], &src[0], len(dst)) } + +//go:noescape +func sqrt32NEON(dst, src *float32, n int) + //go:noescape func add32NEON(dst, a, b *float32, n int) diff --git a/internal/kernels/bin32_generic.go b/internal/kernels/bin32_generic.go index bfb0400..2b44bff 100644 --- a/internal/kernels/bin32_generic.go +++ b/internal/kernels/bin32_generic.go @@ -5,3 +5,6 @@ package kernels // bin32 is the scalar loop on the architectures without float32 elementwise // kernels. dst, a and b have the same length, at least 1. func bin32(op int, dst, a, b []float32) { bin32Go(op, dst, a, b) } + +// sqrt32 is the scalar loop there too. +func sqrt32(dst, src []float32) { sqrt32Go(dst, src) } diff --git a/internal/kernels/bin32_loong64.go b/internal/kernels/bin32_loong64.go index 144dd2c..117c7ae 100644 --- a/internal/kernels/bin32_loong64.go +++ b/internal/kernels/bin32_loong64.go @@ -12,6 +12,19 @@ func bin32(op int, dst, a, b []float32) { run32([...]bin32Func{add32LSX, sub32LSX, mul32LSX, div32LSX}[op], dst, a, b) } +// sqrt32 runs vfsqrt.s (sum_loong64.s) behind the same LASX gate; the scalar +// loop otherwise. dst and src have the same length, at least 1. +func sqrt32(dst, src []float32) { + if !useLASX { + sqrt32Go(dst, src) + return + } + sqrt32LSX(&dst[0], &src[0], len(dst)) +} + +//go:noescape +func sqrt32LSX(dst, src *float32, n int) + //go:noescape func add32LSX(dst, a, b *float32, n int) diff --git a/internal/kernels/bin32_ppc64le.go b/internal/kernels/bin32_ppc64le.go index 2b080b1..f9c33c2 100644 --- a/internal/kernels/bin32_ppc64le.go +++ b/internal/kernels/bin32_ppc64le.go @@ -7,6 +7,13 @@ func bin32(op int, dst, a, b []float32) { run32([...]bin32Func{add32VSX, sub32VSX, mul32VSX, div32VSX}[op], dst, a, b) } +// sqrt32 runs xvsqrtsp (sum_ppc64le.s). dst and src have the same length, at +// least 1. +func sqrt32(dst, src []float32) { sqrt32VSX(&dst[0], &src[0], len(dst)) } + +//go:noescape +func sqrt32VSX(dst, src *float32, n int) + //go:noescape func add32VSX(dst, a, b *float32, n int) diff --git a/internal/kernels/bin32_riscv64.go b/internal/kernels/bin32_riscv64.go index 3cf60f3..d062c71 100644 --- a/internal/kernels/bin32_riscv64.go +++ b/internal/kernels/bin32_riscv64.go @@ -11,6 +11,19 @@ func bin32(op int, dst, a, b []float32) { run32([...]bin32Func{add32RVV, sub32RVV, mul32RVV, div32RVV}[op], dst, a, b) } +// sqrt32 runs vfsqrt.v (sum_riscv64.s) when the CPU reports V; the scalar +// loop otherwise. dst and src have the same length, at least 1. +func sqrt32(dst, src []float32) { + if !useRVV { + sqrt32Go(dst, src) + return + } + sqrt32RVV(&dst[0], &src[0], len(dst)) +} + +//go:noescape +func sqrt32RVV(dst, src *float32, n int) + //go:noescape func add32RVV(dst, a, b *float32, n int) diff --git a/internal/kernels/bin32_s390x.go b/internal/kernels/bin32_s390x.go index 42727e1..7ae4a9c 100644 --- a/internal/kernels/bin32_s390x.go +++ b/internal/kernels/bin32_s390x.go @@ -12,6 +12,19 @@ func bin32(op int, dst, a, b []float32) { run32([...]bin32Func{add32VXE, sub32VXE, mul32VXE, div32VXE}[op], dst, a, b) } +// sqrt32 runs vfsqsb (sum_s390x.s) on a z14 or later; the scalar loop +// otherwise. dst and src have the same length, at least 1. +func sqrt32(dst, src []float32) { + if !useVXE { + sqrt32Go(dst, src) + return + } + sqrt32VXE(&dst[0], &src[0], len(dst)) +} + +//go:noescape +func sqrt32VXE(dst, src *float32, n int) + //go:noescape func add32VXE(dst, a, b *float32, n int) diff --git a/internal/kernels/bin32_test.go b/internal/kernels/bin32_test.go index e777551..e5390e5 100644 --- a/internal/kernels/bin32_test.go +++ b/internal/kernels/bin32_test.go @@ -65,6 +65,39 @@ func TestBin32InPlace(t *testing.T) { } } +// TestSqrt32MatchesScalar checks the vector square root against +// float32(math.Sqrt(float64(x))), bit for bit, including negatives (NaN), +// -0, infinities and subnormals, at every length around the block, and in +// place. +func TestSqrt32MatchesScalar(t *testing.T) { + for n := 0; n <= 70; n++ { + src := operands32(n, 3) + got, want := make([]float32, n), make([]float32, n) + Sqrt32(got, src) + sqrt32Go(want, src) + for i := range got { + if !sameBits32(got[i], want[i]) { + t.Fatalf("n=%d [%d]: sqrt(%v) = %v, scalar %v", n, i, src[i], got[i], want[i]) + } + } + Sqrt32(src, src) + for i := range src { + if !sameBits32(src[i], want[i]) { + t.Fatalf("n=%d [%d]: in place %v, scalar %v", n, i, src[i], want[i]) + } + } + } +} + +func TestSqrt32RefusesAShortSource(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("a short src was accepted") + } + }() + Sqrt32(make([]float32, 4), make([]float32, 3)) +} + func TestBin32RefusesBadInput(t *testing.T) { for _, f := range []func(){ func() { Bin32(Add32, make([]float32, 4), make([]float32, 3), make([]float32, 4)) }, diff --git a/internal/kernels/fence_test.go b/internal/kernels/fence_test.go index 1587dc2..e6155f7 100644 --- a/internal/kernels/fence_test.go +++ b/internal/kernels/fence_test.go @@ -90,6 +90,7 @@ func TestBin32StaysInsideItsOperands(t *testing.T) { for op := Add32; op <= Div32; op++ { Bin32(op, dst, a, b) } + Sqrt32(dst, a) } } } diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index dba30ca..8edd087 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -898,3 +898,39 @@ TEXT ·div32SSE2(SB), NOSPLIT, $0-32 fdone: RET +TEXT ·sqrt32SSE2(SB), NOSPLIT, $0-24 + MOVQ dst+0(FP), DI + MOVQ src+8(FP), SI + MOVQ n+16(FP), CX + qblock: + CMPQ CX, $16 + JL qtail + MOVUPS 0(SI), X0 + MOVUPS 16(SI), X1 + MOVUPS 32(SI), X2 + MOVUPS 48(SI), X3 + SQRTPS X0, X0 + SQRTPS X1, X1 + SQRTPS X2, X2 + SQRTPS X3, X3 + MOVUPS X0, 0(DI) + MOVUPS X1, 16(DI) + MOVUPS X2, 32(DI) + MOVUPS X3, 48(DI) + ADDQ $64, SI + ADDQ $64, DI + SUBQ $16, CX + JMP qblock + qtail: + TESTQ CX, CX + JZ qdone + MOVSS (SI), X0 + SQRTSS X0, X0 + MOVSS X0, (DI) + ADDQ $4, SI + ADDQ $4, DI + DECQ CX + JMP qtail + qdone: + RET + diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index 340c8f3..0bdc449 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -346,6 +346,31 @@ TEXT ·div32NEON(SB), NOSPLIT, $0-32 done: RET +TEXT ·sqrt32NEON(SB), NOSPLIT, $0-24 + MOVD dst+0(FP), R0 + MOVD src+8(FP), R1 + MOVD n+16(FP), R2 + block: + CMP $16, R2 + BLT tail + VLD1.P 64(R1), [V0.S4, V1.S4, V2.S4, V3.S4] + VFSQRT V0.S4, V0.S4 + VFSQRT V1.S4, V1.S4 + VFSQRT V2.S4, V2.S4 + VFSQRT V3.S4, V3.S4 + VST1.P [V0.S4, V1.S4, V2.S4, V3.S4], 64(R0) + SUB $16, R2 + B block + tail: + CBZ R2, done + FMOVS.P 4(R1), F0 + FSQRTS F0, F0 + FMOVS.P F0, 4(R0) + SUB $1, R2 + B tail + done: + RET + TEXT ·maxNEON(SB), NOSPLIT, $0-24 MOVD a+0(FP), R0 MOVD n+8(FP), R1 diff --git a/internal/kernels/sum_loong64.s b/internal/kernels/sum_loong64.s index 9729476..cb279d1 100644 --- a/internal/kernels/sum_loong64.s +++ b/internal/kernels/sum_loong64.s @@ -616,3 +616,38 @@ b8loop: VMOVQ V1, 16(R4) RET +TEXT ·sqrt32LSX(SB), NOSPLIT, $0-24 + MOVV dst+0(FP), R4 + MOVV src+8(FP), R6 + MOVV n+16(FP), R5 +qloop: + MOVV $16, R12 + BLT R5, R12, qtail + VMOVQ 0(R6), V0 + VMOVQ 16(R6), V1 + VMOVQ 32(R6), V2 + VMOVQ 48(R6), V3 + VFSQRTF V0, V4 + VFSQRTF V1, V5 + VFSQRTF V2, V6 + VFSQRTF V3, V7 + VMOVQ V4, 0(R4) + VMOVQ V5, 16(R4) + VMOVQ V6, 32(R4) + VMOVQ V7, 48(R4) + ADDV $64, R4 + ADDV $64, R6 + ADDV $-16, R5 + JMP qloop +qtail: + BEQ R5, qdone + MOVF 0(R6), F1 + SQRTF F1, F1 + MOVF F1, 0(R4) + ADDV $4, R4 + ADDV $4, R6 + ADDV $-1, R5 + JMP qtail +qdone: + RET + diff --git a/internal/kernels/sum_ppc64le.s b/internal/kernels/sum_ppc64le.s index bbc8dd1..78b6b0b 100644 --- a/internal/kernels/sum_ppc64le.s +++ b/internal/kernels/sum_ppc64le.s @@ -858,3 +858,42 @@ b8loop: STXVW4X VS1, (R3)(R14) RET +TEXT ·sqrt32VSX(SB), NOSPLIT, $0-24 + MOVD dst+0(FP), R3 + MOVD src+8(FP), R5 + MOVD n+16(FP), R4 + MOVD $16, R14 + MOVD $32, R15 + MOVD $48, R16 +qloop: + CMP R4, $16 + BLT qtail + LXVW4X (R5)(R0), VS32 + LXVW4X (R5)(R14), VS33 + LXVW4X (R5)(R15), VS34 + LXVW4X (R5)(R16), VS35 + WORD $0xf080022f // XVSQRTSP VS32, VS36 + WORD $0xf0a00a2f // XVSQRTSP VS33, VS37 + WORD $0xf0c0122f // XVSQRTSP VS34, VS38 + WORD $0xf0e01a2f // XVSQRTSP VS35, VS39 + STXVW4X VS36, (R3)(R0) + STXVW4X VS37, (R3)(R14) + STXVW4X VS38, (R3)(R15) + STXVW4X VS39, (R3)(R16) + ADD $64, R3 + ADD $64, R5 + ADD $-16, R4 + BR qloop +qtail: + CMP R4, $0 + BEQ qdone + FMOVS 0(R5), F1 + FSQRTS F1, F1 + FMOVS F1, 0(R3) + ADD $4, R3 + ADD $4, R5 + ADD $-1, R4 + BR qtail +qdone: + RET + diff --git a/internal/kernels/sum_riscv64.s b/internal/kernels/sum_riscv64.s index 461d1fd..f2760a1 100644 --- a/internal/kernels/sum_riscv64.s +++ b/internal/kernels/sum_riscv64.s @@ -289,3 +289,19 @@ TEXT ·blocks8RVV(SB), NOSPLIT, $0-24 VSE32V V2, (X10) RET +TEXT ·sqrt32RVV(SB), NOSPLIT, $0-24 + MOV dst+0(FP), X10 + MOV src+8(FP), X11 + MOV n+16(FP), X13 + loop: + VSETVLI X13, E32, M8, TA, MA, X14 + VLE32V (X11), V0 + VFSQRTV V0, V16 + VSE32V V16, (X10) + SLLI $2, X14, X17 + ADD X17, X10, X10 + ADD X17, X11, X11 + SUB X14, X13, X13 + BNEZ X13, loop + RET + diff --git a/internal/kernels/sum_s390x.s b/internal/kernels/sum_s390x.s index f7af7cb..1e1a84b 100644 --- a/internal/kernels/sum_s390x.s +++ b/internal/kernels/sum_s390x.s @@ -559,3 +559,33 @@ TEXT ·blocks8VXE(SB), NOSPLIT, $0-24 VST V1, 16(R1) RET +TEXT ·sqrt32VXE(SB), NOSPLIT, $0-24 + MOVD dst+0(FP), R1 + MOVD src+8(FP), R2 + MOVD n+16(FP), R4 + loop: + CMP R4, $16 + BLT tail + VLM (R2), V0, V3 + WORD $0xe7000000; BYTE $0x20; BYTE $0xce // VFSQSB V0, V0 + WORD $0xe7110000; BYTE $0x20; BYTE $0xce // VFSQSB V1, V1 + WORD $0xe7220000; BYTE $0x20; BYTE $0xce // VFSQSB V2, V2 + WORD $0xe7330000; BYTE $0x20; BYTE $0xce // VFSQSB V3, V3 + VSTM V0, V3, (R1) + ADD $64, R1 + ADD $64, R2 + SUB $16, R4 + BR loop + tail: + CMP R4, $0 + BEQ done + FMOVS (R2), F0 + FSQRTS F0, F0 + FMOVS F0, (R1) + ADD $4, R1 + ADD $4, R2 + SUB $1, R4 + BR tail + done: + RET +