Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions alloc_bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package ndarray

import "testing"

// The cost of a result, taken apart (#118): the allocation alone, the fill,
// and the operations built on them, at the size of the refbench table.

const allocN = 100_000

var (
sinkF []float64
sinkA *Array
)

func BenchmarkAllocMake(b *testing.B) {
for b.Loop() {
sinkF = make([]float64, allocN)
}
}

func BenchmarkAllocZeros(b *testing.B) {
for b.Loop() {
sinkA, _ = Zeros(allocN)
}
}

func BenchmarkAllocFull(b *testing.B) {
for b.Loop() {
sinkA, _ = Full(1.5, allocN)
}
}

func BenchmarkFillLoop(b *testing.B) {
d := make([]float64, allocN)
for b.Loop() {
for i := range d {
d[i] = 1.5
}
}
sinkF = d
}

func BenchmarkFillCopy(b *testing.B) {
d := make([]float64, allocN)
for b.Loop() {
d[0] = 1.5
for k := 1; k < len(d); k *= 2 {
copy(d[k:], d[:k])
}
}
sinkF = d
}

func BenchmarkAllocAdd(b *testing.B) {
x, _ := Full(1.25, allocN)
y, _ := Full(2.5, allocN)
for b.Loop() {
sinkA, _ = x.Add(y)
}
}

func BenchmarkAllocAddScalar(b *testing.B) {
x, _ := Full(1.25, allocN)
for b.Loop() {
sinkA = x.AddScalar(1.5)
}
}
22 changes: 22 additions & 0 deletions docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,28 @@ issue per row:

Faster, for example: `SqrtInto` (0.51), `fft.FFT` (0.54), `Sum` (0.70), `Exp` (0.89).

### Since the table

Fixed after v0.41.0, measured by `go test -bench` on the same core, not yet
by a new table run. On 10⁵ float64 at load 2.2–2.4, over three interleaved
rounds:
- The scalar forms (`AddScalar` … `DivScalar`, `AddInto` … `DivInto`, and
any Float64 operation with a one-element operand against a contiguous
array of the result's shape) no longer copy the scalar to the result's
size. A strided operand, such as a transpose, still takes the general path. They
stream it through the same SIMD kernel, so the result is bit for bit the
same. `AddScalar` went from 183–203 µs to 95–110 µs, and from 1.6 MB to
0.8 MB per call.
- `Full`, `Ones` and `FullOf` fill by doubling `copy` instead of storing one
element at a time. `Full` went from 131–135 µs to 84–104 µs. `Ones` calls
`Full`. `FullOf`, which fills the other dtypes, was not timed.

What remains of the allocation cause is Go's own cost. A CPU profile of a bare
`make([]float64, 100000)` shows about 60% garbage-collector marking (assists
in the allocating goroutine) and about 23% zeroing. NumPy frees each result
when its last reference goes and reuses that memory warm. In Go, the way to
avoid both costs is to not allocate: use the `*Into` forms or a `Workspace`.

The run was made from the release's own code. The test binary was
cross-compiled from a clean v0.41.0 worktree, the tag was shipped to the host
as a git bundle, and the binary ran from outside the clone so that the tree
Expand Down
8 changes: 6 additions & 2 deletions gen_dtypes.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,11 @@ func main() {
p("")
p("package ndarray")
p("")
p(`import "strconv"`)
p(`import (`)
p(`"strconv"`)
p(``)
p(`"github.com/go-ndarray/ndarray/internal/kernels"`)
p(`)`)
p("")

p("// dtypeOfStore returns the dtype of a storage slice.")
Expand Down Expand Up @@ -130,7 +134,7 @@ func main() {
p("// storage slice of the same dtype.")
p("func fillStore(dst, one any) {")
p("switch d := dst.(type) {")
each("biucf", "case []$T: v := one.([]$T)[0]; for i := range d { d[i] = v }")
each("biucf", "case []$T: kernels.Fill(d, one.([]$T)[0])")
p("}")
p("}")
p("")
Expand Down
44 changes: 44 additions & 0 deletions internal/kernels/parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,50 @@ func RepeatP(k func(dst, a, b []float64), dst, full, rep []float64, repFirst boo
parallelFor(blocks, min(numWorkers(len(dst)), blocks), body)
}

// scalarBlock is the length of the block ScalarP fills with the scalar: long
// enough that a kernel call per block costs little next to the block's work,
// short enough (4 KiB) to stay in L1 for the whole pass.
const scalarBlock = 512

// ScalarP computes dst = full OP s, where s is a single value broadcast over
// full (x + 1.5): it fills one block of scalarBlock copies of s and streams it
// through the elementwise kernel k like RepeatP, so s is never materialised to
// full's size. Each element goes through the same kernel as the materialised
// broadcast, so the result is the same, bit for bit. With sFirst the operands
// are swapped (dst = s OP full). len(dst) == len(full).
func ScalarP(k func(dst, a, b []float64), dst, full []float64, s float64, sFirst bool) {
if len(dst) == 0 {
return
}
// Heap, not stack: the parallel workers share it, so it would escape
// anyway, and a short array then pays for its own length only.
rep := make([]float64, min(scalarBlock, len(dst)))
Fill(rep, s)
whole := len(dst) / len(rep) * len(rep)
RepeatP(k, dst[:whole], full[:whole], rep, sFirst)
if tail := len(dst) - whole; tail > 0 {
if sFirst {
k(dst[whole:], rep[:tail], full[whole:])
} else {
k(dst[whole:], full[whole:], rep[:tail])
}
}
}

// Fill sets every element of dst to v, for every element type. It writes one
// element, then doubles the filled prefix with copy, whose memmove stores
// whole vectors: 3.5× faster than an element-by-element loop on 10⁵ float64
// (Zen 3, #118).
func Fill[T any](dst []T, v T) {
if len(dst) == 0 {
return
}
dst[0] = v
for k := 1; k < len(dst); k *= 2 {
copy(dst[k:], dst[:k])
}
}

// AddP writes a[i]+b[i] into dst[i], parallelised above ParThreshold.
func AddP(dst, a, b []float64) { runBinaryP(addBin, dst, a, b) }

Expand Down
65 changes: 65 additions & 0 deletions internal/kernels/scalar_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package kernels

import (
"math"
"testing"
)

// TestScalarPSameBits: streaming the scalar block by block gives, bit for
// bit, what the kernel gives on the scalar materialised to full length, for
// both operand orders, at every block boundary and across ParThreshold.
func TestScalarPSameBits(t *testing.T) {
ks := map[string]func(dst, a, b []float64){"Add": AddP, "Sub": SubP, "Mul": MulP, "Div": DivP}
const s = 1.0 / 3
for _, n := range []int{0, 1, 7, scalarBlock - 1, scalarBlock, scalarBlock + 1, 3*scalarBlock + 5, ParThreshold + 3} {
full := make([]float64, n)
for i := range full {
full[i] = math.Sin(float64(i)) * 7
}
wide := make([]float64, n)
for i := range wide {
wide[i] = s
}
for name, k := range ks {
for _, sFirst := range []bool{false, true} {
want := make([]float64, n)
if sFirst {
k(want, wide, full)
} else {
k(want, full, wide)
}
got := make([]float64, n)
ScalarP(k, got, full, s, sFirst)
// In place, as AddInto(out = x) does: dst is full itself.
inPlace := append([]float64(nil), full...)
ScalarP(k, inPlace, inPlace, s, sFirst)
for i := range want {
if math.Float64bits(got[i]) != math.Float64bits(want[i]) {
t.Fatalf("%s n=%d sFirst=%v [%d]: got %v, want %v", name, n, sFirst, i, got[i], want[i])
}
if math.Float64bits(inPlace[i]) != math.Float64bits(want[i]) {
t.Fatalf("%s n=%d sFirst=%v in place [%d]: got %v, want %v", name, n, sFirst, i, inPlace[i], want[i])
}
}
}
}
}
}

// TestFill: every element is set, for lengths around the powers of two the
// doubling copy steps through, and nothing past len(dst) is touched.
func TestFill(t *testing.T) {
for _, n := range []int{0, 1, 2, 3, 63, 64, 65, 1000} {
buf := make([]float64, n+1)
buf[n] = -1
Fill(buf[:n], 2.5)
for i := range n {
if buf[i] != 2.5 {
t.Fatalf("n=%d: [%d] = %v", n, i, buf[i])
}
}
if buf[n] != -1 {
t.Fatalf("n=%d: wrote past the end", n)
}
}
}
29 changes: 26 additions & 3 deletions ndarray.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,7 @@ func Full(v float64, shape ...int) (*Array, error) {
if err != nil {
return nil, err
}
for i := range a.data {
a.data[i] = v
}
kernels.Fill(a.data, v)
return a, nil
}

Expand Down Expand Up @@ -661,6 +659,12 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error
if err != nil {
return nil, err
}
if full, s, sFirst, ok := scalarOperand(a, b, shape); ok {
dst := a.alloc(prod(shape), false)
kernels.ScalarP(kernel, dst, full, s, sFirst)
cp := append([]int(nil), shape...)
return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil
}
if full, rep, repFirst, ok := repeatOperands(a, b, shape); ok {
dst := a.alloc(prod(shape), false)
kernels.RepeatP(kernel, dst, full, rep, repFirst)
Expand All @@ -675,6 +679,21 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error
return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil
}

// scalarOperand recognises the broadcast of a single element (x + 1.5, or a
// 1×1 array against a matrix): one operand holds one element, the other is
// contiguous with the full result shape. binOp then streams the value through
// the kernel (kernels.ScalarP) instead of materialising it to the result's size.
func scalarOperand(a, b *Array, shape []int) (full []float64, s float64, sFirst, ok bool) {
isFull := func(f *Array) bool { return f.isContiguous() && sameShape(f.shape, shape) }
switch {
case b.Size() == 1 && isFull(a):
return a.data, b.data[b.offset], false, true
case a.Size() == 1 && isFull(b):
return b.data, a.data[a.offset], true, true
}
return nil, 0, false, false
}

// repeatMin is the shortest repeated operand binOp streams block by block;
// below it a kernel call per block costs more than materialising.
const repeatMin = 64
Expand Down Expand Up @@ -738,6 +757,10 @@ func (a *Array) binOpInto(out, b *Array, kernel func(dst, x, y []float64)) error
return fmt.Errorf("%w: out shape %v != result shape %v",
ErrBroadcast, out.shape, shape)
}
if full, s, sFirst, ok := scalarOperand(a, b, shape); ok {
kernels.ScalarP(kernel, out.data, full, s, sFirst)
return nil
}
x := a.operandFor(shape)
y := b.operandFor(shape)
kernel(out.data, x, y)
Expand Down
87 changes: 87 additions & 0 deletions scalar_operand_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package ndarray

import (
"math"
"testing"
)

// TestScalarOperandPaths: the operations that take binOp's single-element
// path (#118) — a scalar, a 1×1 array, a one-element view with an offset, on
// either side — give the same bits as the element-by-element definition.
func TestScalarOperandPaths(t *testing.T) {
const n = 1001
x, _ := Linspace(-3, 5, n)
xs := x.data // Linspace is contiguous at offset 0
mat, _ := x.Reshape(7, 143)
big, _ := Linspace(0, 10, 11)
view, _ := big.Slice(R(4, 5)) // the element 4.0, at offset 4
oneByOne, _ := Full(4, 1, 1)

ops := []struct {
name string
got func() (*Array, error)
want func(v float64) float64
}{
{"AddScalar", func() (*Array, error) { return x.AddScalar(0.1), nil }, func(v float64) float64 { return v + 0.1 }},
{"SubScalar", func() (*Array, error) { return x.SubScalar(0.1), nil }, func(v float64) float64 { return v - 0.1 }},
{"MulScalar", func() (*Array, error) { return x.MulScalar(0.1), nil }, func(v float64) float64 { return v * 0.1 }},
{"DivScalar", func() (*Array, error) { return x.DivScalar(0.1), nil }, func(v float64) float64 { return v / 0.1 }},
{"scalar-first Div", func() (*Array, error) { return scalarArray(0.1).Div(x) }, func(v float64) float64 { return 0.1 / v }},
{"scalar-first Sub", func() (*Array, error) { return scalarArray(0.1).Sub(x) }, func(v float64) float64 { return 0.1 - v }},
{"view at offset", func() (*Array, error) { return x.Sub(view) }, func(v float64) float64 { return v - 4 }},
{"view first", func() (*Array, error) { return view.Div(x) }, func(v float64) float64 { return 4 / v }},
{"1x1 against matrix", func() (*Array, error) { return mat.Mul(oneByOne) }, func(v float64) float64 { return v * 4 }},
{"DivInto scalar first", func() (*Array, error) {
out, _ := New(n)
return out, scalarArray(0.1).DivInto(out, x)
}, func(v float64) float64 { return 0.1 / v }},
{"SubInto aliasing x", func() (*Array, error) {
y := x.Copy()
return y, y.SubInto(y, scalarArray(0.1))
}, func(v float64) float64 { return v - 0.1 }},
}
for _, op := range ops {
r, err := op.got()
if err != nil {
t.Fatalf("%s: %v", op.name, err)
}
if r.Size() != n {
t.Fatalf("%s: size %d, want %d", op.name, r.Size(), n)
}
for i, g := range r.data[r.offset : r.offset+n] {
if w := op.want(xs[i]); math.Float64bits(g) != math.Float64bits(w) {
t.Fatalf("%s [%d]: got %v, want %v", op.name, i, g, w)
}
}
}
if got := mat.MulScalar(2).Shape(); got[0] != 7 || got[1] != 143 {
t.Fatalf("shape %v, want [7 143]", got)
}
}

// TestScalarOperandAllocs: a scalar operation allocates its result and a
// block of the scalar, never a copy of the scalar at the result's size.
func TestScalarOperandAllocs(t *testing.T) {
x, _ := Linspace(0, 1, 100_000)
var r *Array
bytes := testing.Benchmark(func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
r = x.AddScalar(1.5)
}
}).AllocedBytesPerOp()
if limit := int64(8*100_000 + 8*1024); bytes > limit {
t.Fatalf("AddScalar allocates %d B per call, want at most %d (result + one block)", bytes, limit)
}
_ = r
out, _ := New(100_000)
intoBytes := testing.Benchmark(func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
_ = x.AddInto(out, scalarArray(1.5))
}
}).AllocedBytesPerOp()
if intoBytes > 8*1024 {
t.Fatalf("AddInto with a scalar allocates %d B per call, want at most one block", intoBytes)
}
}
Loading
Loading