From 53c8d762cbd3c7b0b7ce70d931300477c3b17909 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 09:07:45 +0200 Subject: [PATCH] perf: fold contiguous reductions directly; NumPy's pairwise sum for float32/complex A float32 Sum of 2^20 elements took 6.85 ms on POWER9 against 0.23 ms for float64: 6.8 ns an element, while Add was only 1.5x slower. Every non-float64 reduction walked [outer][axisLen][inner] with the reduced axis outside, so a whole-array sum (inner == 1) took a slice and ran a one-iteration loop per element. A contiguous run is now folded directly, as NumPy's add and multiply reductions do: the first element, then the rest added to it, pairwise for float32 and complex (NumPy's FLOAT/CFLOAT_pairwise_sum, loops_utils.h.src v2.5.3), in order for integers and products. POWER9, 1 thread, n=2^20, benchstat over 6 interleaved runs (p=0.002): float32 Sum -76%, int64 -64%, int32 -54%, float64 (control) ~. More accurate too: error vs the exact sum 75 -> 0.87 on 2^16 mixed values, and 2^24 + 1000 ones no longer saturates at 2^24; the new tests fail on the old code. Coverage 100%; compiles for 386 and arm. Also: BenchmarkDTypeSum/Add, a perf.md section, and the roadmap brought up to date (fft, linalg and the Ruby binding shipped; float32 SIMD is what is open). Co-Authored-By: Claude Opus 5.5 --- benchmarks/dtype_bench_test.go | 44 ++++++++++++ docs/perf.md | 36 ++++++++++ docs/plan-ndarray.md | 26 ++++--- sum_pairwise_test.go | 122 +++++++++++++++++++++++++++++++++ typed.go | 116 +++++++++++++++++++++++++++++++ 5 files changed, 334 insertions(+), 10 deletions(-) create mode 100644 benchmarks/dtype_bench_test.go create mode 100644 sum_pairwise_test.go diff --git a/benchmarks/dtype_bench_test.go b/benchmarks/dtype_bench_test.go new file mode 100644 index 0000000..7fdacac --- /dev/null +++ b/benchmarks/dtype_bench_test.go @@ -0,0 +1,44 @@ +package benchmarks + +import ( + "fmt" + "testing" + + nd "github.com/go-ndarray/ndarray" +) + +// The same reduction and elementwise operation on the dtypes that have SIMD +// kernels (float64) and on those that run the pure-Go loops (float32 and the +// integers), at a cache-resident and a memory-bound size. Run them with +// GOMAXPROCS=1 to compare kernels rather than the multicore fan-out. They +// measure what a float32 kernel would be worth before anyone writes one. + +var dtypeBenchTypes = []nd.DType{nd.Float64, nd.Float32, nd.Int64, nd.Int32} + +func BenchmarkDTypeSum(b *testing.B) { + for _, dt := range dtypeBenchTypes { + for _, n := range []int{1 << 14, 1 << 20} { + x := vec(n).AsType(dt) + b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) { + b.SetBytes(int64(n * dt.Size())) + for i := 0; i < b.N; i++ { + _ = x.Sum() + } + }) + } + } +} + +func BenchmarkDTypeAdd(b *testing.B) { + for _, dt := range dtypeBenchTypes { + for _, n := range []int{1 << 14, 1 << 20} { + x, y := vec(n).AsType(dt), vec(n).AsType(dt) + b.Run(fmt.Sprintf("%v/n=%d", dt, n), func(b *testing.B) { + b.SetBytes(int64(n * dt.Size())) + for i := 0; i < b.N; i++ { + _, _ = x.Add(y) + } + }) + } + } +} diff --git a/docs/perf.md b/docs/perf.md index 52128bf..c9d1a5e 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -797,6 +797,42 @@ the morning. Only ratios taken in the same run are comparable. Not measured on Apple M-series, whose machine was shared with other jobs at the time. The bands stay dynamic, so its slower efficiency cores are handled as before. +## Sums of the other dtypes (2026-10-18) + +Measured first, because nothing had: `BenchmarkDTypeSum` and +`BenchmarkDTypeAdd` (`benchmarks/dtype_bench_test.go`) run the same operation +on float64, float32, int64 and int32. On POWER9 (cfarm29, one thread pinned to +the idlest CPU by `hostload -pick 1`, the host 3% busy before and after), a +float32 `Sum` of 2²⁰ elements took **6.85 ms against 0.23 ms for float64**, +6.8 ns an element, while `Add` was only 1.5× slower than float64. A missing +SIMD kernel explains a factor of 4 to 8, not 30. + +The cause was the reduction loop. Every non-float64 reduction walked the +`[outer][axisLen][inner]` layout with the reduced axis outside and the kept +elements inside, so a whole-array sum (`inner` = 1) took a slice and ran a +one-iteration loop per element. A contiguous run is now folded directly, as +NumPy's add and multiply reductions do: the first element, then the rest +added to it, **pairwise for float32 and the complex dtypes** (NumPy's +`FLOAT/CFLOAT_pairwise_sum`, `loops_utils.h.src` at v2.5.3: eight +accumulators, blocks of 128, halved above), in order for the integers (exact +in any order) and for products (NumPy multiplies in order). + +| POWER9, 1 thread, n = 2²⁰ | before | after | | +|---|---:|---:|---:| +| float32 `Sum` | 6.850 ms | 1.637 ms | −76% | +| int64 `Sum` | 5.195 ms | 1.870 ms | −64% | +| int32 `Sum` | 6.170 ms | 2.868 ms | −54% | +| float64 `Sum` (its own kernels; the control) | 225.7 µs | 225.7 µs | ~ | + +Six runs each, old and new binaries alternating, `benchstat` p = 0.002. The +float32 sum is also more accurate: on 2¹⁶ values of mixed sign and magnitude +its error against the exact sum (`math/big`) fell from 75 to 0.87, and 2²⁴ +followed by 1000 ones no longer saturates at 2²⁴ (`sum_pairwise_test.go`). + +Left: float32 has no SIMD kernel yet (still 7× float64 for `Sum`), and the +integer sums convert the whole array to int64 before adding (NumPy's +accumulator type), an extra pass and allocation. + ## SIMD coverage - **amd64 (SSE2)** ships hand-vectorized `sum` (4-accumulator `ADDPD`), `sqrt` diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index 51c907e..13ee7d4 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -153,16 +153,22 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). first-class bool dtype (so masks are real booleans) remains Phase 2. - **Phase 2 — dtypes — DONE.** Thirteen dtypes (bool, signed and unsigned integers, float32/64, complex64/128) as a runtime property of `Array`, - with NumPy 2.x promotion; see [design-dtypes.md](design-dtypes.md). Next: - SIMD kernels for `float32`, and the `fft` bridge. -- **Phase 3 — broadcasting ufuncs.** A general ufunc framework over views - (the axis-reduction primitives above are the first step); more elementwise - and reduction ufuncs. -- **Phase 4 — linear algebra.** `MatMul`/`Dot`, transpose-aware GEMM, - decompositions; SIMD/blocked kernels. -- **Phase 5 — Ruby binding.** Expose the array type to Ruby through - [go-embedded-ruby](https://github.com/go-embedded-ruby/ruby) as a cgo-free - alternative to `Numo::NArray`. + with NumPy 2.x promotion; see [design-dtypes.md](design-dtypes.md). The + `fft` bridge shipped in v0.10.0 (package `fft`, over go-fft). Still open: + SIMD kernels for `float32` (float32 runs the pure-Go code today). +- **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic, + comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`, + `Where` and the per-axis reductions, in every dtype. Not done: a public + framework for user-defined ufuncs (`np.frompyfunc`, `np.vectorize`). +- **Phase 4 — linear algebra — DONE (v0.11.0).** `MatMul`/`Dot` on a + panel-packed GEMM with SIMD micro-kernels on six architectures, and package + `linalg` (`numpy.linalg`: solve, inverse, determinants, Cholesky, QR, SVD, + eigenproblems, least squares, norms); see [linalg.md](linalg.md) for what + is left there. +- **Phase 5 — Ruby binding — DONE.** + [go-embedded-ruby](https://github.com/go-embedded-ruby/ruby) binds the array + as `NDArray` (since its v0.5.0, the API the go-ndarray playground runs in the + browser), a cgo-free alternative to `Numo::NArray`. ## 5. Decisions diff --git a/sum_pairwise_test.go b/sum_pairwise_test.go new file mode 100644 index 0000000..372ad57 --- /dev/null +++ b/sum_pairwise_test.go @@ -0,0 +1,122 @@ +package ndarray + +import ( + "math" + "math/big" + "testing" +) + +// Sums along a contiguous axis of float32 and complex arrays are pairwise, as +// NumPy's are (FLOAT/CFLOAT_pairwise_sum, numpy/_core/src/umath/ +// loops_utils.h.src). The expectations below come from exact arithmetic, not +// from running the code. + +// TestFloat32SumDoesNotSaturate: 2^24 followed by 1000 ones. Added in order +// in float32, every 2^24 + 1 rounds back to 2^24, so the sum stays 2^24. +// Pairwise, the ones are summed among themselves first (1000, exact), and +// 2^24 + 1000 is representable (float32's spacing at 2^24 is 2). +func TestFloat32SumDoesNotSaturate(t *testing.T) { + const n = 1000 + v := make([]float32, n+1) + v[0] = 1 << 24 + for i := 1; i <= n; i++ { + v[i] = 1 + } + a := must(FromSlice(v, len(v))) + if got, want := a.Sum(), float64(1<<24+n); got != want { + t.Errorf("Sum = %v, want %v (in order it would be %v)", got, want, float64(1<<24)) + } + // The same along the last axis of a 2-D array, row by row. + two := must(FromSlice(append(append([]float32{}, v...), v...), 2, len(v))) + rows, ok := Data[float32](must(two.SumAxis(1, false))) + if !ok || len(rows) != 2 { + t.Fatalf("SumAxis(1) of a 2-row float32 array: %v, %v", rows, ok) + } + for i, got := range rows { + if got != 1<<24+n { + t.Errorf("row %d: %v, want %v", i, got, 1<<24+n) + } + } + c := make([]complex64, n+1) + c[0] = complex(1<<24, -(1 << 24)) + for i := 1; i <= n; i++ { + c[i] = complex(1, -1) + } + got := Item[complex64](must(FromSlice(c, len(c))).SumAll()) + if want := complex64(complex(1<<24+n, -(1<<24 + n))); got != want { + t.Errorf("complex64 sum = %v, want %v", got, want) + } +} + +// TestFloat32SumError compares the sum of 2^16 float32 values of mixed sign +// and magnitude with the exact sum (math/big): pairwise must be far closer +// than a sum in order. +func TestFloat32SumError(t *testing.T) { + const n = 1 << 16 + v := make([]float32, n) + x := uint32(12345) + for i := range v { + x = x*1664525 + 1013904223 // a fixed LCG: reproducible, no seed to drift + v[i] = float32(int32(x)>>8) / float32(1<<8) * float32(1+int(x%7)) + } + exact := new(big.Float).SetPrec(256) + var inOrder float32 + for _, f := range v { + exact.Add(exact, new(big.Float).SetFloat64(float64(f))) + inOrder += f + } + e, _ := exact.Float64() + got := must(FromSlice(v, n)).Sum() + errPairwise, errInOrder := math.Abs(got-e), math.Abs(float64(inOrder)-e) + t.Logf("exact %v: pairwise %v (error %g), in order %v (error %g)", e, got, errPairwise, inOrder, errInOrder) + if errPairwise*10 > errInOrder { + t.Errorf("pairwise error %g is not ten times below the in-order error %g", errPairwise, errInOrder) + } +} + +// TestProdAlongANonContiguousAxis: a product along axis 0 of a 2-D integer +// array keeps the row-by-row loop (the reduced axis is not contiguous), and +// accumulates in int64 as NumPy does for int32. +func TestProdAlongANonContiguousAxis(t *testing.T) { + a := must(FromSlice([]int32{1, 2, 3, 4, 5, -6}, 2, 3)) + p := must(a.ProdAxis(0, false)) + got, ok := Data[int64](p) + if !ok || len(got) != 3 || got[0] != 4 || got[1] != 10 || got[2] != -18 { + t.Errorf("ProdAxis(0) = %v %v, want int64 [4 10 -18]", p.DType(), got) + } +} + +// TestPairwiseSumBranches covers each length class of NumPy's scheme and +// checks it against the exact sum of small integers (exact in float32), and +// that a sum of only -0 stays -0. +func TestPairwiseSumBranches(t *testing.T) { + for _, n := range []int{1, 2, 7, 8, 9, 15, 16, 127, 128, 129, 130, 1000, 4099} { + v := make([]float32, n) + c := make([]complex64, n) + want := 0 + for i := range v { + v[i] = float32(i%13 - 6) + c[i] = complex(float32(i%5), float32(-(i % 3))) + want += i%13 - 6 + } + if got := must(FromSlice(v, n)).Sum(); got != float64(want) { + t.Errorf("n=%d: float32 sum %v, want %d", n, got, want) + } + var wre, wim int + for i := range c { + wre += i % 5 + wim -= i % 3 + } + if got := Item[complex64](must(FromSlice(c, n)).SumAll()); got != complex(float32(wre), float32(wim)) { + t.Errorf("n=%d: complex64 sum %v, want (%d%+di)", n, got, wre, wim) + } + } + negZero := float32(math.Copysign(0, -1)) + if got := must(FromSlice([]float32{negZero, negZero, negZero}, 3)).Sum(); !math.Signbit(got) { + t.Errorf("a sum of -0 is %v, want -0", got) + } + nz := complex(negZero, negZero) + if got := Item[complex64](must(FromSlice([]complex64{nz, nz}, 2)).SumAll()); !math.Signbit(float64(real(got))) || !math.Signbit(float64(imag(got))) { + t.Errorf("a complex sum of -0 is %v, want -0-0i", got) + } +} diff --git a/typed.go b/typed.go index 33d9e4e..f2956b8 100644 --- a/typed.go +++ b/typed.go @@ -345,6 +345,16 @@ const ( // reduceNum sums or multiplies along the axis. func reduceNum[T numT](op reduceCode, dst, src []T, outer, axisLen, inner int) { + if inner == 1 { + // The reduced axis is contiguous (the last axis, or the whole + // array): fold each run directly. The loop below would take a + // slice and run a one-iteration loop per element there, 6.8 ns an + // element for a float32 sum on POWER9. + for o := 0; o < outer; o++ { + dst[o] = foldRun(op, src[o*axisLen:(o+1)*axisLen]) + } + return + } for o := 0; o < outer; o++ { row := dst[o*inner : (o+1)*inner] copy(row, src[o*axisLen*inner:(o*axisLen+1)*inner]) @@ -363,6 +373,112 @@ func reduceNum[T numT](op reduceCode, dst, src []T, outer, axisLen, inner int) { } } +// foldRun reduces one contiguous run (len(s) > 0) as NumPy's add and +// multiply reductions do: the result starts as the first element and the +// rest is added to it, pairwise for a floating sum (pairwiseSum), in order +// otherwise. Integer sums are exact in any order, and NumPy multiplies in +// order. +func foldRun[T numT](op reduceCode, s []T) T { + acc := s[0] + rest := s[1:] + if op == rSum { + switch r := any(rest).(type) { + case []float32: // Float64 sums never come here: they have kernels + return any(any(acc).(float32) + pairwiseSum(r)).(T) + case []complex64: + return any(any(acc).(complex64) + pairwiseSumComplex(r)).(T) + case []complex128: + return any(any(acc).(complex128) + pairwiseSumComplex(r)).(T) + } + for _, v := range rest { + acc += v + } + return acc + } + for _, v := range rest { + acc *= v + } + return acc +} + +// pwBlock is NumPy's PW_BLOCKSIZE: runs up to this length are summed with +// eight accumulators, longer ones are halved. +const pwBlock = 128 + +// pairwiseSum is NumPy's FLOAT_pairwise_sum +// (numpy/_core/src/umath/loops_utils.h.src, v2.5.3): rounding error O(lg n) +// instead of O(n), in the same order of additions. +func pairwiseSum[F float32](a []F) F { + n := len(a) + switch { + case n < 8: + // From -0, so that a sum of only -0 stays -0. + res := F(math.Copysign(0, -1)) + for _, v := range a { + res += v + } + return res + case n <= pwBlock: + var r [8]F + copy(r[:], a[:8]) + i := 8 + for ; i < n-n%8; i += 8 { + r[0] += a[i] + r[1] += a[i+1] + r[2] += a[i+2] + r[3] += a[i+3] + r[4] += a[i+4] + r[5] += a[i+5] + r[6] += a[i+6] + r[7] += a[i+7] + } + res := ((r[0] + r[1]) + (r[2] + r[3])) + ((r[4] + r[5]) + (r[6] + r[7])) + for ; i < n; i++ { + res += a[i] + } + return res + } + n2 := n / 2 + n2 -= n2 % 8 + return pairwiseSum(a[:n2]) + pairwiseSum(a[n2:]) +} + +// pairwiseSumComplex is NumPy's CFLOAT/CDOUBLE_pairwise_sum: the same scheme +// over the interleaved real and imaginary parts, counted in floats (two per +// element), so four accumulators per part and blocks of 64 elements. +func pairwiseSumComplex[C complex64 | complex128](a []C) C { + n := 2 * len(a) // NumPy's n counts floats + switch { + case n < 8: + re, im := math.Copysign(0, -1), math.Copysign(0, -1) + var acc C = C(complex(re, im)) + for _, v := range a { + acc += v + } + return acc + case n <= pwBlock: + // r[0], r[2], r[4], r[6] hold real parts and r[1], r[3], r[5], r[7] + // imaginary parts, of elements 0..3 of each block of four. + var r [4]C + copy(r[:], a[:4]) + i := 4 + for ; i < len(a)-len(a)%4; i += 4 { + r[0] += a[i] + r[1] += a[i+1] + r[2] += a[i+2] + r[3] += a[i+3] + } + res := (r[0] + r[1]) + (r[2] + r[3]) + for ; i < len(a); i++ { + res += a[i] + } + return res + } + n2 := n / 2 + n2 -= n2 % 8 + return pairwiseSumComplex(a[:n2/2]) + pairwiseSumComplex(a[n2/2:]) +} + // reduceOrdered takes the maximum or minimum along the axis; for floats a NaN // wins, as in NumPy. func reduceOrdered[T realT](op reduceCode, dst, src []T, outer, axisLen, inner int) {