Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions broadcast_repeat_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package ndarray

import (
"testing"

"github.com/go-ndarray/ndarray/internal/kernels"
)

// TestBroadcastRepeat checks the streamed broadcast (a full operand and a
// repeated one, see repeatOperands) against an element-by-element oracle, for
// the non-commutative ops in both operand orders, several suffix shapes, a
// repeated operand too short to stream, and with the block loop forced onto
// several cores.
func TestBroadcastRepeat(t *testing.T) {
grid := func(seed int, shape ...int) *Array {
n := 1
for _, d := range shape {
n *= d
}
d := make([]float64, n)
for i := range d {
d[i] = float64((i*13+seed)%29) - 14.5
}
return mustArr(t, ok(FromData(d, shape...)))
}
cases := []struct {
name string
full, rep *Array
streamable bool
}{
{"matrix, row", grid(1, 9, 70), grid(2, 1, 70), true},
{"matrix, 1-D", grid(3, 5, 80), grid(4, 80), true},
{"3-D, 2-D", grid(5, 4, 3, 30), grid(6, 3, 30), true},
{"3-D, row of 1-D", grid(7, 2, 3, 100), grid(8, 1, 1, 100), true},
{"short row", grid(9, 6, 10), grid(10, 1, 10), false},
}
ops := []struct {
name string
f func(x, y *Array) (*Array, error)
g func(x, y float64) float64
}{
{"sub", (*Array).Sub, func(x, y float64) float64 { return x - y }},
{"div", (*Array).Div, func(x, y float64) float64 { return x / y }},
}
run := func() {
for _, c := range cases {
if _, _, _, got := repeatOperands(c.full, c.rep, c.full.shape); got != c.streamable {
t.Errorf("%s: streamable = %v, want %v", c.name, got, c.streamable)
}
for _, op := range ops {
for _, swap := range []bool{false, true} {
x, y := c.full, c.rep
if swap {
x, y = y, x
}
got, err := op.f(x, y)
if err != nil {
t.Fatal(err)
}
if !sameShape(got.shape, c.full.shape) {
t.Fatalf("%s %s: shape %v", c.name, op.name, got.shape)
}
m := c.rep.Size()
for i, v := range got.data {
fv, rv := c.full.data[i], c.rep.data[i%m]
want := op.g(fv, rv)
if swap {
want = op.g(rv, fv)
}
if v != want {
t.Fatalf("%s %s swap=%v [%d]: %v, want %v", c.name, op.name, swap, i, v, want)
}
}
}
}
}
}
run()
saved := kernels.ParThreshold
kernels.ParThreshold = 64
defer func() { kernels.ParThreshold = saved }()
run()
}
14 changes: 14 additions & 0 deletions docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,20 @@ Zen 3, 16 cores pinned, minimum of three interleaved runs:
| MaxAxis(1) | 601 µs | **93 µs** | 189 µs | 2.0× |

On M4: MaxAxis(1) ×6–8, SumAxis(0) ×1.9.
## Broadcasting a row without materialising it (2026-10-04)

`M + row` (1024×1024 plus 1×1024) lost to NumPy 0.26× on the Zen 3 even with a
`Workspace`: the broadcast path copied the row out to a full 8 MiB matrix
before adding. When one operand has the full shape and the other, leading 1s
dropped, is a suffix of it (a row, a 2-D mean subtracted from a 3-D stack), the
second simply repeats in blocks of its own length, so `binOp` now streams the
live slices block by block through the SIMD kernel (`kernels.RepeatP`), split
across cores. Below 64 elements per block the old path is kept.

| BroadcastAdd 1024²+row, Zen 3 | heap | `Workspace` | NumPy 1 thread |
|----|--:|--:|--:|
| before | 3.1 ms | 1.76 ms | 460 µs |
| after | **0.84 ms** | **114–196 µs** | 460 µs |

## SIMD coverage

Expand Down
27 changes: 27 additions & 0 deletions internal/kernels/parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,33 @@ func runBinaryP(k binaryKernel, dst, a, b []float64) {
// IEEE-754 op — no reduction grouping), so the result matches the serial scalar
// computation for every input.

// RepeatP computes dst = full OP rep, where rep is repeated along full in
// consecutive blocks of len(rep) — a broadcast of rep over the leading axes,
// such as a matrix plus a row. Each block is one call of the elementwise
// kernel k on live slices, so rep is never materialised to full's size; the
// blocks are split across cores above ParThreshold. With repFirst the operands
// are swapped (dst = rep OP full), for the non-commutative ops. len(dst) ==
// len(full) is a multiple of len(rep) >= 1.
func RepeatP(k func(dst, a, b []float64), dst, full, rep []float64, repFirst bool) {
m := len(rep)
blocks := len(dst) / m
body := func(lo, hi int) {
for b := lo; b < hi; b++ {
s := b * m
if repFirst {
k(dst[s:s+m], rep, full[s:s+m])
} else {
k(dst[s:s+m], full[s:s+m], rep)
}
}
}
if len(dst) < ParThreshold {
body(0, blocks)
return
}
parallelFor(blocks, min(numWorkers(len(dst)), blocks), body)
}

// AddP writes a[i]+b[i] into dst[i], parallelised above ParThreshold.
func AddP(dst, a, b []float64) { runBinaryP(addBin, dst, a, b) }

Expand Down
35 changes: 35 additions & 0 deletions ndarray.go
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,12 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error
if err != nil {
return nil, err
}
if full, rep, repFirst, ok := repeatOperands(a, b, shape); ok {
dst := a.alloc(prod(shape), false)
kernels.RepeatP(kernel, dst, full, rep, repFirst)
cp := append([]int(nil), shape...)
return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil
}
x := a.operandFor(shape)
y := b.operandFor(shape)
dst := a.alloc(prod(shape), false)
Expand All @@ -634,6 +640,35 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error
return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil
}

// repeatMin is the shortest repeated operand binOp streams block by block;
// below it a kernel call per block costs more than materialising.
const repeatMin = 64

// repeatOperands recognises the broadcast that is a plain repetition: one
// operand is contiguous with the full result shape, the other is contiguous
// and its shape, leading 1s dropped, is a suffix of the result shape (a matrix
// plus a row, a 3-D stack minus a 2-D mean). The second then repeats in
// consecutive blocks of its own length, and binOp can stream the live slices
// instead of materialising the broadcast (see kernels.RepeatP).
func repeatOperands(a, b *Array, shape []int) (full, rep []float64, repFirst, ok bool) {
isRep := func(r *Array) bool {
s := r.shape
for len(s) > 0 && s[0] == 1 {
s = s[1:]
}
return r.isContiguous() && r.Size() >= repeatMin &&
len(s) <= len(shape) && sameShape(s, shape[len(shape)-len(s):])
}
isFull := func(f *Array) bool { return f.isContiguous() && sameShape(f.shape, shape) }
switch {
case isFull(a) && isRep(b):
return a.data, b.data, false, true
case isFull(b) && isRep(a):
return b.data, a.data, true, true
}
return nil, nil, false, false
}

// Add returns the elementwise sum a+b with broadcasting.
func (a *Array) Add(b *Array) (*Array, error) { return a.binOp(b, kernels.AddP) }

Expand Down
Loading