From 4c571b2a69906c3bb2323d15b6cbaba2f4bf218b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 11:06:35 +0200 Subject: [PATCH] fix: reject overflowing shapes, count huge-step slices, broadcast 0-d views Three crashes or silent wrong results found while probing edge cases: - Element counts were multiplied without an overflow check. New(1<<62, 4) returned an array with Size 0; FromData and Reshape accepted (1<<62+1, 4) for 4 elements because the product wrapped to exactly 4, and At(1, 0) then panicked; New(1<<59, 8) panicked in makeslice. Shapes whose float64 storage would not fit in an int are now rejected with ErrShapeMismatch, as numpy does ("array is too big"), including a broadcast result built from two valid shapes. - Slice computed ceil((stop-start)/step) as (stop-start+step-1)/step, which overflows for a step near MaxInt: x[0:5:MaxInt] was empty instead of [x0]. - A 0-d view at a non-zero offset (x.Slice(A(2))) is not contiguous, so elementwise ops sent it down the broadcasting path, which read shape[n-1] with n == 0 and panicked with "index out of range [-1]". Co-Authored-By: Claude Opus 5.5 (1M context) --- ndarray.go | 31 ++++++++++++++++- overflow_test.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++ slicing.go | 9 +++-- 3 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 overflow_test.go diff --git a/ndarray.go b/ndarray.go index 99adef2..e1a4b60 100644 --- a/ndarray.go +++ b/ndarray.go @@ -14,6 +14,7 @@ package ndarray import ( "errors" "fmt" + "math" "strconv" "strings" @@ -45,12 +46,28 @@ func prod(dims []int) int { return n } -// validateShape reports whether every dimension is non-negative. +// maxSize is the largest element count whose float64 storage size in bytes +// still fits in an int — numpy's "array is too big" bound. +const maxSize = math.MaxInt / 8 + +// validateShape reports whether every dimension is non-negative and the +// element count is representable. The count check multiplies the non-zero +// dimensions with an overflow guard, so a shape whose product would wrap (e.g. +// (1<<62+1, 4), which wraps to exactly 4) is rejected rather than accepted +// with a wrong Size; like numpy, a zero dimension does not excuse the others. func validateShape(shape []int) error { + n := 1 for _, d := range shape { if d < 0 { return fmt.Errorf("%w: negative dimension %d", ErrShapeMismatch, d) } + if d == 0 { + continue + } + if d > maxSize/n { + return fmt.Errorf("%w: shape %v is too big", ErrShapeMismatch, shape) + } + n *= d } return nil } @@ -327,6 +344,9 @@ func (a *Array) Reshape(shape ...int) (*Array, error) { if err != nil { return nil, err } + if err := validateShape(shape); err != nil { + return nil, err + } if prod(shape) != a.Size() { return nil, fmt.Errorf("%w: cannot reshape size %d into %v", ErrShapeMismatch, a.Size(), shape) @@ -427,6 +447,10 @@ func broadcastShape(s1, s2 []int) ([]int, error) { return nil, fmt.Errorf("%w: %v vs %v", ErrBroadcast, s1, s2) } } + // Two representable shapes can broadcast to one that is not. + if err := validateShape(out); err != nil { + return nil, err + } return out, nil } @@ -450,6 +474,11 @@ func (a *Array) broadcastTo(shape []int) []float64 { if len(out) == 0 { return out } + if n == 0 { + // 0-d target: the receiver is a 0-d (possibly offset) view too. + out[0] = a.data[a.offset] + return out + } // Fast path: the innermost axis is materialised with unit stride (it is // either a real, non-broadcast last axis or a broadcast length-1 one). Then // each contiguous inner run is a single copy() — for a broadcast last axis the diff --git a/overflow_test.go b/overflow_test.go new file mode 100644 index 0000000..195b2ba --- /dev/null +++ b/overflow_test.go @@ -0,0 +1,90 @@ +package ndarray + +import ( + "errors" + "math" + "testing" +) + +// TestShapeSizeOverflow checks that a shape whose element count overflows int +// is rejected instead of wrapping. Before the fix prod() wrapped silently: +// New(1<<62, 4) returned an array of Size 0, and FromData/Reshape accepted +// (1<<62+1, 4) for 4 elements because the product wrapped to exactly 4, after +// which At(1, 0) panicked. numpy raises "array is too big" for all of these. +func TestShapeSizeOverflow(t *testing.T) { + huge := [][]int{ + {1 << 62, 4}, + {1<<62 + 1, 4}, + {1 << 59, 8}, + {math.MaxInt, 2}, + {0, 1 << 62, 1 << 62}, // numpy rejects this too, despite the zero + } + for _, s := range huge { + if _, err := New(s...); !errors.Is(err, ErrShapeMismatch) { + t.Errorf("New(%v) err = %v, want ErrShapeMismatch", s, err) + } + } + if _, err := FromData([]float64{1, 2, 3, 4}, 1<<62+1, 4); !errors.Is(err, ErrShapeMismatch) { + t.Errorf("FromData wrapped shape err = %v", err) + } + a := mustArr(t, ok(Arange(0, 4, 1))) + if _, err := a.Reshape(1<<62+1, 4); !errors.Is(err, ErrShapeMismatch) { + t.Errorf("Reshape wrapped shape err = %v", err) + } + if _, err := a.Reshape(-1, 1<<62+1, 4); !errors.Is(err, ErrShapeMismatch) { + t.Errorf("Reshape wrapped shape with -1 err = %v", err) + } + // Two valid shapes can broadcast to an unrepresentable one. + col := &Array{data: []float64{1}, shape: []int{1 << 40, 1}, strides: []int{0, 0}} + row := &Array{data: []float64{1}, shape: []int{1, 1 << 40}, strides: []int{0, 0}} + if _, err := col.Add(row); !errors.Is(err, ErrShapeMismatch) { + t.Errorf("broadcast to (2^40, 2^40) err = %v", err) + } + // Large but representable shapes are still fine. + if _, err := New(0, 1<<40); err != nil { + t.Errorf("New(0, 1<<40) err = %v", err) + } +} + +// TestSliceHugeStep checks the element count of a slice whose |step| exceeds +// the axis length. Before the fix ceil((stop-start)/step) was computed as +// (stop-start+step-1)/step, which overflows for a step near MaxInt and gave an +// empty view; numpy gives one element (Python a[0:5:sys.maxsize] == a[0:1]). +func TestSliceHugeStep(t *testing.T) { + x := mustArr(t, ok(Arange(0, 6, 1))) + for _, c := range []struct { + name string + ix Index + want []float64 + }{ + {"0:5:MaxInt", Rng(0, 5, math.MaxInt), []float64{0}}, + {"4:0:-MaxInt", Rng(4, 0, -math.MaxInt), []float64{4}}, + {"::-MaxInt", Step(-math.MaxInt), []float64{5}}, + {"::MaxInt", Step(math.MaxInt), []float64{0}}, + {"::MinInt", Step(math.MinInt), []float64{5}}, + {"5:0:MaxInt", Rng(5, 0, math.MaxInt), []float64{}}, + } { + s, err := x.Slice(c.ix) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + wantData(t, s, []int{len(c.want)}, c.want) + } +} + +// TestZeroDimViewBroadcast checks elementwise ops on a 0-d view that does not +// start at offset 0 (x[2] of a 1-D x). Such a view is not "contiguous", so it +// takes the broadcasting path, which indexed shape[n-1] with n == 0 and +// panicked with "index out of range [-1]". +func TestZeroDimViewBroadcast(t *testing.T) { + x := mustArr(t, ok(Arange(0, 6, 1))) + s := mustArr(t, ok(x.Slice(A(2)))) + r := mustArr(t, ok(s.Add(s))) + wantData(t, r, nil, []float64{4}) // Shape() of a 0-d array is nil + r = mustArr(t, ok(s.Mul(x))) + wantData(t, r, []int{6}, []float64{0, 2, 4, 6, 8, 10}) + m := mustArr(t, ok(s.MaskSelect(s))) + wantData(t, m, []int{1}, []float64{2}) + w := mustArr(t, ok(Where(s, s, x))) + wantData(t, w, []int{6}, []float64{2, 2, 2, 2, 2, 2}) +} diff --git a/slicing.go b/slicing.go index 92237ab..78ba38c 100644 --- a/slicing.go +++ b/slicing.go @@ -99,9 +99,11 @@ func (ix Index) resolveRange(length int) (first, count, step int, err error) { if ix.hasStop { stop = normBound(ix.stop, length, 0, length) } + // ceil((stop-first)/step), written so that it cannot overflow for a + // step near MaxInt (stop-first+step-1 would). count = 0 if stop > first { - count = (stop - first + step - 1) / step + count = (stop-first-1)/step + 1 } return first, count, step, nil } @@ -115,9 +117,12 @@ func (ix Index) resolveRange(length int) (first, count, step int, err error) { if ix.hasStop { stop = normBound(ix.stop, length, -1, length-1) } + // ceil((first-stop)/-step) without overflow. For step == MinInt, -step + // wraps to MinInt; the quotient is then 0 (the dividend is smaller in + // magnitude), which still yields the right count of 1. count = 0 if first > stop { - count = (first - stop - step - 1) / (-step) + count = (first-stop-1)/(-step) + 1 } return first, count, step, nil }