From d5fc1e327b4e77be334bfdf125ed8a22316908cd Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 15:21:30 +0200 Subject: [PATCH] perf: stream a repeated broadcast operand instead of materialising it M + row (1024x1024 plus 1x1024) lost to NumPy 0.26x on a Zen 3 even in a Workspace, because the broadcast path copied the row out to a full 8 MiB matrix before adding. When one operand has the full result shape and the other, leading 1s dropped, is a suffix of it, the second just repeats in blocks of its own length: binOp now hands the live slices to the SIMD kernel block by block (kernels.RepeatP), split across cores. Blocks under 64 elements keep the old path. Zen 3, 16 cores pinned, BroadcastAdd 1024^2 + row: heap 3.1 ms -> 0.84 ms Workspace 1.76 ms -> 114-196 us (NumPy 1 thread: 460 us) The new test checks Sub and Div in both operand orders, matrix+row, matrix+1-D, 3-D minus 2-D, 3-D plus a (1,1,n) row and a too-short row, serially and with the block loop forced parallel, against an element by element oracle; swapping the operands in RepeatP makes it fail. Co-Authored-By: Claude Opus 5.5 --- broadcast_repeat_test.go | 83 ++++++++++++++++++++++++++++++++++++ docs/perf.md | 14 ++++++ internal/kernels/parallel.go | 27 ++++++++++++ ndarray.go | 35 +++++++++++++++ 4 files changed, 159 insertions(+) create mode 100644 broadcast_repeat_test.go diff --git a/broadcast_repeat_test.go b/broadcast_repeat_test.go new file mode 100644 index 0000000..bdb317f --- /dev/null +++ b/broadcast_repeat_test.go @@ -0,0 +1,83 @@ +package ndarray + +import ( + "testing" + + "github.com/go-ndarray/ndarray/internal/kernels" +) + +// TestBroadcastRepeat checks the streamed broadcast (a full operand and a +// repeated one, see repeatOperands) against an element-by-element oracle, for +// the non-commutative ops in both operand orders, several suffix shapes, a +// repeated operand too short to stream, and with the block loop forced onto +// several cores. +func TestBroadcastRepeat(t *testing.T) { + grid := func(seed int, shape ...int) *Array { + n := 1 + for _, d := range shape { + n *= d + } + d := make([]float64, n) + for i := range d { + d[i] = float64((i*13+seed)%29) - 14.5 + } + return mustArr(t, ok(FromData(d, shape...))) + } + cases := []struct { + name string + full, rep *Array + streamable bool + }{ + {"matrix, row", grid(1, 9, 70), grid(2, 1, 70), true}, + {"matrix, 1-D", grid(3, 5, 80), grid(4, 80), true}, + {"3-D, 2-D", grid(5, 4, 3, 30), grid(6, 3, 30), true}, + {"3-D, row of 1-D", grid(7, 2, 3, 100), grid(8, 1, 1, 100), true}, + {"short row", grid(9, 6, 10), grid(10, 1, 10), false}, + } + ops := []struct { + name string + f func(x, y *Array) (*Array, error) + g func(x, y float64) float64 + }{ + {"sub", (*Array).Sub, func(x, y float64) float64 { return x - y }}, + {"div", (*Array).Div, func(x, y float64) float64 { return x / y }}, + } + run := func() { + for _, c := range cases { + if _, _, _, got := repeatOperands(c.full, c.rep, c.full.shape); got != c.streamable { + t.Errorf("%s: streamable = %v, want %v", c.name, got, c.streamable) + } + for _, op := range ops { + for _, swap := range []bool{false, true} { + x, y := c.full, c.rep + if swap { + x, y = y, x + } + got, err := op.f(x, y) + if err != nil { + t.Fatal(err) + } + if !sameShape(got.shape, c.full.shape) { + t.Fatalf("%s %s: shape %v", c.name, op.name, got.shape) + } + m := c.rep.Size() + for i, v := range got.data { + fv, rv := c.full.data[i], c.rep.data[i%m] + want := op.g(fv, rv) + if swap { + want = op.g(rv, fv) + } + if v != want { + t.Fatalf("%s %s swap=%v [%d]: %v, want %v", c.name, op.name, swap, i, v, want) + } + } + } + } + } + } + run() + saved := kernels.ParThreshold + kernels.ParThreshold = 64 + defer func() { kernels.ParThreshold = saved }() + run() +} diff --git a/docs/perf.md b/docs/perf.md index 04f2d7a..380efb4 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -436,6 +436,20 @@ Zen 3, 16 cores pinned, minimum of three interleaved runs: | MaxAxis(1) | 601 µs | **93 µs** | 189 µs | 2.0× | On M4: MaxAxis(1) ×6–8, SumAxis(0) ×1.9. +## Broadcasting a row without materialising it (2026-10-04) + +`M + row` (1024×1024 plus 1×1024) lost to NumPy 0.26× on the Zen 3 even with a +`Workspace`: the broadcast path copied the row out to a full 8 MiB matrix +before adding. When one operand has the full shape and the other, leading 1s +dropped, is a suffix of it (a row, a 2-D mean subtracted from a 3-D stack), the +second simply repeats in blocks of its own length, so `binOp` now streams the +live slices block by block through the SIMD kernel (`kernels.RepeatP`), split +across cores. Below 64 elements per block the old path is kept. + +| BroadcastAdd 1024²+row, Zen 3 | heap | `Workspace` | NumPy 1 thread | +|----|--:|--:|--:| +| before | 3.1 ms | 1.76 ms | 460 µs | +| after | **0.84 ms** | **114–196 µs** | 460 µs | ## SIMD coverage diff --git a/internal/kernels/parallel.go b/internal/kernels/parallel.go index 5b4991d..835406b 100644 --- a/internal/kernels/parallel.go +++ b/internal/kernels/parallel.go @@ -83,6 +83,33 @@ func runBinaryP(k binaryKernel, dst, a, b []float64) { // IEEE-754 op — no reduction grouping), so the result matches the serial scalar // computation for every input. +// RepeatP computes dst = full OP rep, where rep is repeated along full in +// consecutive blocks of len(rep) — a broadcast of rep over the leading axes, +// such as a matrix plus a row. Each block is one call of the elementwise +// kernel k on live slices, so rep is never materialised to full's size; the +// blocks are split across cores above ParThreshold. With repFirst the operands +// are swapped (dst = rep OP full), for the non-commutative ops. len(dst) == +// len(full) is a multiple of len(rep) >= 1. +func RepeatP(k func(dst, a, b []float64), dst, full, rep []float64, repFirst bool) { + m := len(rep) + blocks := len(dst) / m + body := func(lo, hi int) { + for b := lo; b < hi; b++ { + s := b * m + if repFirst { + k(dst[s:s+m], rep, full[s:s+m]) + } else { + k(dst[s:s+m], full[s:s+m], rep) + } + } + } + if len(dst) < ParThreshold { + body(0, blocks) + return + } + parallelFor(blocks, min(numWorkers(len(dst)), blocks), body) +} + // AddP writes a[i]+b[i] into dst[i], parallelised above ParThreshold. func AddP(dst, a, b []float64) { runBinaryP(addBin, dst, a, b) } diff --git a/ndarray.go b/ndarray.go index 1cf47a9..14ea223 100644 --- a/ndarray.go +++ b/ndarray.go @@ -626,6 +626,12 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error if err != nil { return nil, err } + if full, rep, repFirst, ok := repeatOperands(a, b, shape); ok { + dst := a.alloc(prod(shape), false) + kernels.RepeatP(kernel, dst, full, rep, repFirst) + cp := append([]int(nil), shape...) + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil + } x := a.operandFor(shape) y := b.operandFor(shape) dst := a.alloc(prod(shape), false) @@ -634,6 +640,35 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil } +// repeatMin is the shortest repeated operand binOp streams block by block; +// below it a kernel call per block costs more than materialising. +const repeatMin = 64 + +// repeatOperands recognises the broadcast that is a plain repetition: one +// operand is contiguous with the full result shape, the other is contiguous +// and its shape, leading 1s dropped, is a suffix of the result shape (a matrix +// plus a row, a 3-D stack minus a 2-D mean). The second then repeats in +// consecutive blocks of its own length, and binOp can stream the live slices +// instead of materialising the broadcast (see kernels.RepeatP). +func repeatOperands(a, b *Array, shape []int) (full, rep []float64, repFirst, ok bool) { + isRep := func(r *Array) bool { + s := r.shape + for len(s) > 0 && s[0] == 1 { + s = s[1:] + } + return r.isContiguous() && r.Size() >= repeatMin && + len(s) <= len(shape) && sameShape(s, shape[len(shape)-len(s):]) + } + isFull := func(f *Array) bool { return f.isContiguous() && sameShape(f.shape, shape) } + switch { + case isFull(a) && isRep(b): + return a.data, b.data, false, true + case isFull(b) && isRep(a): + return b.data, a.data, true, true + } + return nil, nil, false, false +} + // Add returns the elementwise sum a+b with broadcasting. func (a *Array) Add(b *Array) (*Array, error) { return a.binOp(b, kernels.AddP) }