Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ Bound: elementwise `+ − × ÷` and `Sqrt` are correctly rounded (one IEEE oper

Every element of `MatMul`/`Dot` satisfies |Ĉᵢⱼ − Cᵢⱼ| ≤ γ_k·(|A|·|B|)ᵢⱼ, whatever the summation order, with or without FMA. Both bounds are from Higham, *Accuracy and Stability of Numerical Algorithms*, §3.1 and §4.2.
Oracle: `math/big` at 1024 bits for sums and inner products. `TestSumDepth`, `TestAccuracyBoundSum` and `TestAccuracyBoundMatMul` check them; the difference is taken in `big.Float`, so the oracle is never rounded. `Exp`/`Log` are checked against a 300-bit reference in go-simd/floats.
Exact: the defaults are still the exact ports, NumPy's pairwise order for `Sum` included, so results match NumPy bit for bit where they did before. When a faster default lands, that arithmetic stays available behind an explicit option.
Exact: the defaults are still the exact ports, NumPy's pairwise order for `Sum` included, so results match NumPy bit for bit where they did before. When a faster default lands, that arithmetic stays available behind an explicit option. For `fft`, that option exists already: `fft.Options{Exact: true}` asks go-fft for its reference arithmetic, and `FMA: true` for its fused twiddles.

## Goroutines

Expand Down
6 changes: 3 additions & 3 deletions bench/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ go 1.27.3

require (
github.com/go-ndarray/ndarray v0.0.0
github.com/go-scikit/refbench v0.1.2
github.com/go-scikit/refbench v0.3.0
)

require (
github.com/go-fft/fft v0.28.11 // indirect
github.com/go-fft/fft v0.29.0 // indirect
github.com/go-fleettools/fleettools v0.81.0 // indirect
github.com/go-simd/floats v0.15.0 // indirect
github.com/go-simd/floats v0.17.0 // indirect
golang.org/x/sys v0.49.0 // indirect
)

Expand Down
12 changes: 6 additions & 6 deletions bench/go.sum
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
github.com/go-fft/fft v0.28.11 h1:ZfT78fSmPjZU2E9PgkSI/0f0E3sesYpl13GHmIFzRGc=
github.com/go-fft/fft v0.28.11/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s=
github.com/go-fft/fft v0.29.0 h1:WtJb5K3dmbLWREvvZ50od0fxtjIBiXrg7mVybz/nPoA=
github.com/go-fft/fft v0.29.0/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s=
github.com/go-fleettools/fleettools v0.81.0 h1:UnMflmNw5b82Z+7Ums7UVaW5RrGuACCVmSwytwT1hxw=
github.com/go-fleettools/fleettools v0.81.0/go.mod h1:y/o3p2HZAvto9J5G5gtPxhN45zgvaYFAXCY9RCK/lQw=
github.com/go-scikit/refbench v0.1.2 h1:bzD0LhsC1UA6HzG1FK8TvQdIYnPib4rEibdUkgVeOfA=
github.com/go-scikit/refbench v0.1.2/go.mod h1:PhJxvLwXiQeKLP2f5ndcb9k5k4PTvU1rSFJ/MKRCrU0=
github.com/go-simd/floats v0.15.0 h1:0joQTqse6+Ccl78FETksCrrBGdeLhy7ZsxRJRNU6yLU=
github.com/go-simd/floats v0.15.0/go.mod h1:Aq5a6oY5pUqm9Hpi+iV2nrNmaaUb0rwHvUHIerXWSvA=
github.com/go-scikit/refbench v0.3.0 h1:ng9b3FRwOWyarkoHHLar1bs43TWi9TydZeO29Yr9iSw=
github.com/go-scikit/refbench v0.3.0/go.mod h1:PhJxvLwXiQeKLP2f5ndcb9k5k4PTvU1rSFJ/MKRCrU0=
github.com/go-simd/floats v0.17.0 h1:Dv0ZmZ3k0IavvHwQgstpUz+TeIf/gGJ9HEGQ/UART8Q=
github.com/go-simd/floats v0.17.0/go.mod h1:Aq5a6oY5pUqm9Hpi+iV2nrNmaaUb0rwHvUHIerXWSvA=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
105 changes: 105 additions & 0 deletions fft/arith_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package fft

import (
"errors"
"math"
"math/cmplx"
"testing"

gofft "github.com/go-fft/fft"
"github.com/go-ndarray/ndarray"
)

// TestArithmeticReachesGoFFT: Exact and FMA arrive in the go-fft Options of
// every kind of transform (c2c, r2c, c2r, herm, iherm) and of the repeated-axis
// path, read from what is actually handed to go-fft. Today Exact equals the
// default and FMA acts only on amd64 with FMA3, so the results alone could not
// show a dropped field.
func TestArithmeticReachesGoFFT(t *testing.T) {
var got []gofft.Options
defer func(h func(gofft.Options)) { handed = h }(handed)
handed = func(o gofft.Options) { got = append(got, o) }
x, err := ndarray.Arange(0, 16, 1)
if err != nil {
t.Fatal(err)
}
m, _ := x.Reshape(4, 4)
for _, o := range []Options{{Exact: true}, {FMA: true}} {
for _, c := range []struct {
name string
f func(*ndarray.Array, Options) (*ndarray.Array, error)
a *ndarray.Array
axes []int
}{
{"FFT", FFTWith, x, nil}, {"RFFT", RFFTWith, x, nil}, {"IRFFT", IRFFTWith, x, nil},
{"HFFT", HFFTWith, x, nil}, {"IHFFT", IHFFTWith, x, nil},
{"FFTN repeated axis", FFTNWith, m, []int{1, 1}},
} {
got = got[:0]
o.Axes = c.axes
if _, err := c.f(c.a, o); err != nil {
t.Fatal(err)
}
if len(got) == 0 {
t.Fatalf("%s %+v: nothing reached go-fft", c.name, o)
}
for _, g := range got {
if g.Exact != o.Exact || g.FMA != o.FMA {
t.Fatalf("%s %+v: go-fft got Exact=%v FMA=%v", c.name, o, g.Exact, g.FMA)
}
}
}
}
}

// TestArithmeticOptions: Exact and FMA reach go-fft. Exact gives today's
// default bits (go-fft's default is its reference arithmetic); FMA stays
// within the transform's error bound of the default; asking for both is
// refused; and a repeated axis (the sequential path) carries them too.
func TestArithmeticOptions(t *testing.T) {
x, err := ndarray.Arange(0, 48, 1)
if err != nil {
t.Fatal(err)
}
x, _ = x.Sin().Reshape(6, 8)
for _, c := range []struct {
name string
f func(*ndarray.Array, Options) (*ndarray.Array, error)
axes []int
}{
{"FFTN", FFTNWith, nil},
{"RFFTN", RFFTNWith, nil},
{"FFTN repeated axis", FFTNWith, []int{1, 1}},
} {
def, err := c.f(x, Options{Axes: c.axes})
if err != nil {
t.Fatal(err)
}
exact, err := c.f(x, Options{Axes: c.axes, Exact: true})
if err != nil {
t.Fatal(err)
}
fma, err := c.f(x, Options{Axes: c.axes, FMA: true})
if err != nil {
t.Fatal(err)
}
d, _ := ndarray.Data[complex128](def)
e, _ := ndarray.Data[complex128](exact)
f, _ := ndarray.Data[complex128](fma)
peak := 0.0
for _, v := range d {
peak = math.Max(peak, cmplx.Abs(v))
}
for i := range d {
if e[i] != d[i] {
t.Fatalf("%s: Exact element %d = %v, default %v", c.name, i, e[i], d[i])
}
if cmplx.Abs(f[i]-d[i]) > 64*0x1p-52*peak {
t.Fatalf("%s: FMA element %d = %v, default %v", c.name, i, f[i], d[i])
}
}
if _, err := c.f(x, Options{Axes: c.axes, Exact: true, FMA: true}); !errors.Is(err, ErrArgs) {
t.Fatalf("%s: Exact and FMA together gave %v, want ErrArgs", c.name, err)
}
}
}
56 changes: 41 additions & 15 deletions fft/fft.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,30 @@ type Options struct {

// Norm scales the result.
Norm Norm

// Exact asks go-fft for its reference arithmetic (go-fft Options.Exact):
// the bits the transforms computed up to go-fft v0.28.16, whatever a later
// default becomes inside the stated error bound. Today the default is that
// arithmetic, so Exact changes nothing yet.
Exact bool

// FMA lets go-fft fuse its twiddle products (go-fft Options.FMA): faster on
// amd64 CPUs with FMA3 and as accurate, but not the default's bits. The
// float32 and complex64 transforms ignore it. Exact and FMA together are
// refused with ErrArgs.
FMA bool
}

// engine is the go-fft Options of a transform along axes, scaled by norm,
// with the arithmetic o asks for.
func (o Options) engine(axes []int, norm Norm) gofft.Options {
return gofft.Options{Axes: axes, Norm: norm, Exact: o.Exact, FMA: o.FMA}
}

// handed sees every go-fft Options right before the call it is passed to; a
// test replaces it to check what actually reaches go-fft.
var handed = func(gofft.Options) {}

// kind is the family of a transform.
type kind uint8

Expand Down Expand Up @@ -196,6 +218,9 @@ const (

// run resolves the arguments, pads or truncates the input, and calls go-fft.
func run(a *ndarray.Array, o Options, k kind, inverse bool, r rank) (*ndarray.Array, error) {
if o.Exact && o.FMA {
return nil, fmt.Errorf("%w: Exact and FMA ask for two arithmetics", ErrArgs)
}
if (k == r2c || k == iherm) && a.DType().IsComplex() {
return nil, fmt.Errorf("%w: a real transform of a %v array", ErrArgs, a.DType())
}
Expand All @@ -208,7 +233,7 @@ func run(a *ndarray.Array, o Options, k kind, inverse bool, r rank) (*ndarray.Ar
return nil, err
}
if dup {
return sequential(a, axes, out, o.Norm, k, inverse)
return sequential(a, axes, out, o, k, inverse)
}
// The input is cut or padded to the lengths the transform reads: for the
// half-spectrum inverses the last axis holds out/2+1 bins.
Expand All @@ -222,21 +247,22 @@ func run(a *ndarray.Array, o Options, k kind, inverse bool, r rank) (*ndarray.Ar
}
single := a.DType() == ndarray.Float32 || a.DType() == ndarray.Complex64
shape := x.Shape()
g := o.engine(axes, o.Norm)
switch k {
case c2c:
return complexTransform(x, shape, axes, o.Norm, inverse, single), nil
return complexTransform(x, shape, g, inverse, single), nil
case r2c:
return realForward(x, shape, axes, o.Norm, single), nil
return realForward(x, shape, g, single), nil
case c2r:
shape[axes[len(axes)-1]] = out[len(out)-1]
return realInverse(x, shape, axes, o.Norm, single), nil
return realInverse(x, shape, g, single), nil
case herm:
// numpy: hfft(a, n) = irfft(conj(a), n) scaled as a forward transform.
shape[axes[len(axes)-1]] = out[len(out)-1]
return realInverse(x.Conj(), shape, axes, opposite(o.Norm), single), nil
return realInverse(x.Conj(), shape, o.engine(axes, opposite(o.Norm)), single), nil
}
// iherm: ihfft(a, n) = conj(rfft(a, n)) scaled as an inverse transform.
return realForward(x, shape, axes, opposite(o.Norm), single).Conj(), nil
return realForward(x, shape, o.engine(axes, opposite(o.Norm)), single).Conj(), nil
}

// sequential runs a multi-axis transform one axis at a time, in the order
Expand All @@ -245,9 +271,9 @@ func run(a *ndarray.Array, o Options, k kind, inverse bool, r rank) (*ndarray.Ar
// refuses a repeated axis.) fftn goes from the last axis to the first; rfftn
// does the real transform along the last axis first, then the others; irfftn
// does the others first and the real inverse last.
func sequential(a *ndarray.Array, axes, out []int, norm Norm, k kind, inverse bool) (*ndarray.Array, error) {
func sequential(a *ndarray.Array, axes, out []int, o Options, k kind, inverse bool) (*ndarray.Array, error) {
one := func(x *ndarray.Array, i int, k kind) (*ndarray.Array, error) {
return run(x, Options{S: []int{out[i]}, Axes: []int{axes[i]}, Norm: norm}, k, inverse, oneD)
return run(x, Options{S: []int{out[i]}, Axes: []int{axes[i]}, Norm: o.Norm, Exact: o.Exact, FMA: o.FMA}, k, inverse, oneD)
}
last := len(axes) - 1
x := a
Expand Down Expand Up @@ -377,8 +403,8 @@ func resize(a *ndarray.Array, ax, n int) *ndarray.Array {
}

// complexTransform runs FFTN/IFFTN along axes.
func complexTransform(x *ndarray.Array, shape, axes []int, norm Norm, inverse, single bool) *ndarray.Array {
o := gofft.Options{Axes: axes, Norm: norm}
func complexTransform(x *ndarray.Array, shape []int, o gofft.Options, inverse, single bool) *ndarray.Array {
handed(o)
if single {
d, _ := ndarray.Data[complex64](x.AsType(ndarray.Complex64))
var r []complex64
Expand All @@ -401,10 +427,10 @@ func complexTransform(x *ndarray.Array, shape, axes []int, norm Norm, inverse, s

// realForward runs RFFTN along axes; the last axis of the result holds
// n/2+1 bins.
func realForward(x *ndarray.Array, shape, axes []int, norm Norm, single bool) *ndarray.Array {
o := gofft.Options{Axes: axes, Norm: norm}
func realForward(x *ndarray.Array, shape []int, o gofft.Options, single bool) *ndarray.Array {
handed(o)
outShape := append([]int(nil), shape...)
last := axes[len(axes)-1]
last := o.Axes[len(o.Axes)-1]
outShape[last] = shape[last]/2 + 1
if single {
d, _ := ndarray.Data[float32](x.Real().AsType(ndarray.Float32))
Expand All @@ -415,8 +441,8 @@ func realForward(x *ndarray.Array, shape, axes []int, norm Norm, single bool) *n
}

// realInverse runs IRFFTN along axes into a real array of the given shape.
func realInverse(x *ndarray.Array, shape, axes []int, norm Norm, single bool) *ndarray.Array {
o := gofft.Options{Axes: axes, Norm: norm}
func realInverse(x *ndarray.Array, shape []int, o gofft.Options, single bool) *ndarray.Array {
handed(o)
if single {
d, _ := ndarray.Data[complex64](x.AsType(ndarray.Complex64))
return wrap(gofft.IRFFTN32With(d, shape, o), shape)
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/go-ndarray/ndarray
go 1.27.2

require (
github.com/go-fft/fft v0.28.11
github.com/go-simd/floats v0.15.0
github.com/go-fft/fft v0.29.0
github.com/go-simd/floats v0.17.0
)

require golang.org/x/sys v0.49.0 // indirect
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
github.com/go-fft/fft v0.28.11 h1:ZfT78fSmPjZU2E9PgkSI/0f0E3sesYpl13GHmIFzRGc=
github.com/go-fft/fft v0.28.11/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s=
github.com/go-simd/floats v0.15.0 h1:0joQTqse6+Ccl78FETksCrrBGdeLhy7ZsxRJRNU6yLU=
github.com/go-simd/floats v0.15.0/go.mod h1:Aq5a6oY5pUqm9Hpi+iV2nrNmaaUb0rwHvUHIerXWSvA=
github.com/go-fft/fft v0.29.0 h1:WtJb5K3dmbLWREvvZ50od0fxtjIBiXrg7mVybz/nPoA=
github.com/go-fft/fft v0.29.0/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s=
github.com/go-simd/floats v0.17.0 h1:Dv0ZmZ3k0IavvHwQgstpUz+TeIf/gGJ9HEGQ/UART8Q=
github.com/go-simd/floats v0.17.0/go.mod h1:Aq5a6oY5pUqm9Hpi+iV2nrNmaaUb0rwHvUHIerXWSvA=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
7 changes: 3 additions & 4 deletions internal/kernels/asmgen/arm64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@
// bit-identical to the scalar oracle; the kernels package validates it to a
// tight relative tolerance against the oracle and against numpy.
//
// (A max-reduction SIMD kernel is intentionally not shipped: NEON FMAX's NaN
// propagation differs from the scalar `if v > m` oracle, so a bit-identical
// vector max is not expressible without a NaN pre-scan; Max stays on the
// associative scalar parallel path.)
// (No max-reduction kernel is generated here: Max and Min come from
// go-simd/floats, whose NEON extrema kernels (extrema_arm64.s there) use
// FMAX/FMIN, which propagate NaN and order -0 below +0 as Go's max/min do.)
package main

import (
Expand Down
4 changes: 2 additions & 2 deletions internal/kernels/asmgen/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@
// go-asmgen to (re)produce the committed .s files via `go generate`.
module github.com/go-ndarray/ndarray/internal/kernels/asmgen

go 1.27.2
go 1.27.3

require github.com/go-asmgen/asmgen v0.23.0
require github.com/go-asmgen/asmgen v0.23.2
4 changes: 2 additions & 2 deletions internal/kernels/asmgen/go.sum
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
github.com/go-asmgen/asmgen v0.23.0 h1:K0L4gQ/CqK7gwuVrCWfrYU8S/OEsUt4qNftkjHIchXU=
github.com/go-asmgen/asmgen v0.23.0/go.mod h1:FHWXZt1MrejOwSShl4tPYjI2GPEfNAJuXJUX6Z+ruyQ=
github.com/go-asmgen/asmgen v0.23.2 h1:7/0fIoF0iCuGJXzwkJbyFspshDh1CSxVf5IdKfiAWeo=
github.com/go-asmgen/asmgen v0.23.2/go.mod h1:e1V8AfWIOlOi+GEr9DLaG6T5ui2PaHVPb7kp3C3IB48=
Loading