Skip to content

Commit ae279b5

Browse files
tannevaledclaude
andauthored
perf: stream a repeated broadcast operand instead of materialising it (#14)
M + row (1024x1024 plus 1x1024) lost to NumPy 0.26x on a Zen 3 even in a Workspace, because the broadcast path copied the row out to a full 8 MiB matrix before adding. When one operand has the full result shape and the other, leading 1s dropped, is a suffix of it, the second just repeats in blocks of its own length: binOp now hands the live slices to the SIMD kernel block by block (kernels.RepeatP), split across cores. Blocks under 64 elements keep the old path. Zen 3, 16 cores pinned, BroadcastAdd 1024^2 + row: heap 3.1 ms -> 0.84 ms Workspace 1.76 ms -> 114-196 us (NumPy 1 thread: 460 us) The new test checks Sub and Div in both operand orders, matrix+row, matrix+1-D, 3-D minus 2-D, 3-D plus a (1,1,n) row and a too-short row, serially and with the block loop forced parallel, against an element by element oracle; swapping the operands in RepeatP makes it fail. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 90d0a5c commit ae279b5

4 files changed

Lines changed: 159 additions & 0 deletions

File tree

‎broadcast_repeat_test.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package ndarray
2+
3+
import (
4+
"testing"
5+
6+
"github.com/go-ndarray/ndarray/internal/kernels"
7+
)
8+
9+
// TestBroadcastRepeat checks the streamed broadcast (a full operand and a
10+
// repeated one, see repeatOperands) against an element-by-element oracle, for
11+
// the non-commutative ops in both operand orders, several suffix shapes, a
12+
// repeated operand too short to stream, and with the block loop forced onto
13+
// several cores.
14+
func TestBroadcastRepeat(t *testing.T) {
15+
grid := func(seed int, shape ...int) *Array {
16+
n := 1
17+
for _, d := range shape {
18+
n *= d
19+
}
20+
d := make([]float64, n)
21+
for i := range d {
22+
d[i] = float64((i*13+seed)%29) - 14.5
23+
}
24+
return mustArr(t, ok(FromData(d, shape...)))
25+
}
26+
cases := []struct {
27+
name string
28+
full, rep *Array
29+
streamable bool
30+
}{
31+
{"matrix, row", grid(1, 9, 70), grid(2, 1, 70), true},
32+
{"matrix, 1-D", grid(3, 5, 80), grid(4, 80), true},
33+
{"3-D, 2-D", grid(5, 4, 3, 30), grid(6, 3, 30), true},
34+
{"3-D, row of 1-D", grid(7, 2, 3, 100), grid(8, 1, 1, 100), true},
35+
{"short row", grid(9, 6, 10), grid(10, 1, 10), false},
36+
}
37+
ops := []struct {
38+
name string
39+
f func(x, y *Array) (*Array, error)
40+
g func(x, y float64) float64
41+
}{
42+
{"sub", (*Array).Sub, func(x, y float64) float64 { return x - y }},
43+
{"div", (*Array).Div, func(x, y float64) float64 { return x / y }},
44+
}
45+
run := func() {
46+
for _, c := range cases {
47+
if _, _, _, got := repeatOperands(c.full, c.rep, c.full.shape); got != c.streamable {
48+
t.Errorf("%s: streamable = %v, want %v", c.name, got, c.streamable)
49+
}
50+
for _, op := range ops {
51+
for _, swap := range []bool{false, true} {
52+
x, y := c.full, c.rep
53+
if swap {
54+
x, y = y, x
55+
}
56+
got, err := op.f(x, y)
57+
if err != nil {
58+
t.Fatal(err)
59+
}
60+
if !sameShape(got.shape, c.full.shape) {
61+
t.Fatalf("%s %s: shape %v", c.name, op.name, got.shape)
62+
}
63+
m := c.rep.Size()
64+
for i, v := range got.data {
65+
fv, rv := c.full.data[i], c.rep.data[i%m]
66+
want := op.g(fv, rv)
67+
if swap {
68+
want = op.g(rv, fv)
69+
}
70+
if v != want {
71+
t.Fatalf("%s %s swap=%v [%d]: %v, want %v", c.name, op.name, swap, i, v, want)
72+
}
73+
}
74+
}
75+
}
76+
}
77+
}
78+
run()
79+
saved := kernels.ParThreshold
80+
kernels.ParThreshold = 64
81+
defer func() { kernels.ParThreshold = saved }()
82+
run()
83+
}

‎docs/perf.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,20 @@ Zen 3, 16 cores pinned, minimum of three interleaved runs:
436436
| MaxAxis(1) | 601 µs | **93 µs** | 189 µs | 2.0× |
437437

438438
On M4: MaxAxis(1) ×6–8, SumAxis(0) ×1.9.
439+
## Broadcasting a row without materialising it (2026-10-04)
440+
441+
`M + row` (1024×1024 plus 1×1024) lost to NumPy 0.26× on the Zen 3 even with a
442+
`Workspace`: the broadcast path copied the row out to a full 8 MiB matrix
443+
before adding. When one operand has the full shape and the other, leading 1s
444+
dropped, is a suffix of it (a row, a 2-D mean subtracted from a 3-D stack), the
445+
second simply repeats in blocks of its own length, so `binOp` now streams the
446+
live slices block by block through the SIMD kernel (`kernels.RepeatP`), split
447+
across cores. Below 64 elements per block the old path is kept.
448+
449+
| BroadcastAdd 1024²+row, Zen 3 | heap | `Workspace` | NumPy 1 thread |
450+
|----|--:|--:|--:|
451+
| before | 3.1 ms | 1.76 ms | 460 µs |
452+
| after | **0.84 ms** | **114–196 µs** | 460 µs |
439453

440454
## SIMD coverage
441455

‎internal/kernels/parallel.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,33 @@ func runBinaryP(k binaryKernel, dst, a, b []float64) {
8383
// IEEE-754 op — no reduction grouping), so the result matches the serial scalar
8484
// computation for every input.
8585

86+
// RepeatP computes dst = full OP rep, where rep is repeated along full in
87+
// consecutive blocks of len(rep) — a broadcast of rep over the leading axes,
88+
// such as a matrix plus a row. Each block is one call of the elementwise
89+
// kernel k on live slices, so rep is never materialised to full's size; the
90+
// blocks are split across cores above ParThreshold. With repFirst the operands
91+
// are swapped (dst = rep OP full), for the non-commutative ops. len(dst) ==
92+
// len(full) is a multiple of len(rep) >= 1.
93+
func RepeatP(k func(dst, a, b []float64), dst, full, rep []float64, repFirst bool) {
94+
m := len(rep)
95+
blocks := len(dst) / m
96+
body := func(lo, hi int) {
97+
for b := lo; b < hi; b++ {
98+
s := b * m
99+
if repFirst {
100+
k(dst[s:s+m], rep, full[s:s+m])
101+
} else {
102+
k(dst[s:s+m], full[s:s+m], rep)
103+
}
104+
}
105+
}
106+
if len(dst) < ParThreshold {
107+
body(0, blocks)
108+
return
109+
}
110+
parallelFor(blocks, min(numWorkers(len(dst)), blocks), body)
111+
}
112+
86113
// AddP writes a[i]+b[i] into dst[i], parallelised above ParThreshold.
87114
func AddP(dst, a, b []float64) { runBinaryP(addBin, dst, a, b) }
88115

‎ndarray.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,12 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error
626626
if err != nil {
627627
return nil, err
628628
}
629+
if full, rep, repFirst, ok := repeatOperands(a, b, shape); ok {
630+
dst := a.alloc(prod(shape), false)
631+
kernels.RepeatP(kernel, dst, full, rep, repFirst)
632+
cp := append([]int(nil), shape...)
633+
return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil
634+
}
629635
x := a.operandFor(shape)
630636
y := b.operandFor(shape)
631637
dst := a.alloc(prod(shape), false)
@@ -634,6 +640,35 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error
634640
return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil
635641
}
636642

643+
// repeatMin is the shortest repeated operand binOp streams block by block;
644+
// below it a kernel call per block costs more than materialising.
645+
const repeatMin = 64
646+
647+
// repeatOperands recognises the broadcast that is a plain repetition: one
648+
// operand is contiguous with the full result shape, the other is contiguous
649+
// and its shape, leading 1s dropped, is a suffix of the result shape (a matrix
650+
// plus a row, a 3-D stack minus a 2-D mean). The second then repeats in
651+
// consecutive blocks of its own length, and binOp can stream the live slices
652+
// instead of materialising the broadcast (see kernels.RepeatP).
653+
func repeatOperands(a, b *Array, shape []int) (full, rep []float64, repFirst, ok bool) {
654+
isRep := func(r *Array) bool {
655+
s := r.shape
656+
for len(s) > 0 && s[0] == 1 {
657+
s = s[1:]
658+
}
659+
return r.isContiguous() && r.Size() >= repeatMin &&
660+
len(s) <= len(shape) && sameShape(s, shape[len(shape)-len(s):])
661+
}
662+
isFull := func(f *Array) bool { return f.isContiguous() && sameShape(f.shape, shape) }
663+
switch {
664+
case isFull(a) && isRep(b):
665+
return a.data, b.data, false, true
666+
case isFull(b) && isRep(a):
667+
return b.data, a.data, true, true
668+
}
669+
return nil, nil, false, false
670+
}
671+
637672
// Add returns the elementwise sum a+b with broadcasting.
638673
func (a *Array) Add(b *Array) (*Array, error) { return a.binOp(b, kernels.AddP) }
639674

0 commit comments

Comments
 (0)