From be9694c2f27b9053cec585920c0559cb6f446d11 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 2 Nov 2026 12:40:37 +0100 Subject: [PATCH 1/2] kernels: CORE-MATH's correctly rounded logf, with an AVX2 kernel logCR transcribes CORE-MATH's binary32 logf (Sibidanov and Zimmermann, MIT): a 64-entry table and a degree-3 polynomial in float64 bound the result; a degree-8 polynomial decides the 762 750 positive inputs whose bounds round apart. Products are converted to float64 explicitly so no architecture fuses them. TestLogCRMatchesLog32 finds it equal to log32, already proven correctly rounded, on all 2^32 inputs. log32CRAVX2 runs the fast path on 8 floats at a time (VGATHERDPD for the tables, unfused VMULPD/VADDPD in logCR's order) and hands back each block it cannot finish to logCR. Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/amd64/gen.go | 103 +++++++++++++++ internal/kernels/expcr_other.go | 3 + internal/kernels/logcr.go | 138 +++++++++++++++++++++ internal/kernels/logcr_amd64.go | 52 ++++++++ internal/kernels/logcr_amd64_test.go | 91 ++++++++++++++ internal/kernels/logcr_fence_amd64_test.go | 20 +++ internal/kernels/logcr_test.go | 127 +++++++++++++++++++ internal/kernels/sum_amd64.s | 89 +++++++++++++ 8 files changed, 623 insertions(+) create mode 100644 internal/kernels/logcr.go create mode 100644 internal/kernels/logcr_amd64.go create mode 100644 internal/kernels/logcr_amd64_test.go create mode 100644 internal/kernels/logcr_fence_amd64_test.go create mode 100644 internal/kernels/logcr_test.go diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index f1bd76e..4bf0918 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -56,6 +56,7 @@ func main() { f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) f.Add(amd64.FeatureProbe("hasAVX2", amd64.AVX2)) f.Add(exp32CRKernel()) + f.Add(log32CRKernel()) f.Add(blocks8Kernel()) f.Add(bin32Kernel("add32SSE2", "ADDPS", "ADDSS")) f.Add(bin32Kernel("sub32SSE2", "SUBPS", "SUBSS")) @@ -446,6 +447,108 @@ func exp32CRKernel() *emit.Function { return b.Func() } +// log32CRKernel builds log32CRAVX2(dst, src *float32, n int, k +// *logCRConsts) int: logCR's fast path (logcr.go, CORE-MATH's logf) on eight +// float32 at a time, n a positive multiple of 8. Per block: +// +// - the integer part in 32-bit lanes: m = ux & 0x7fffff, j = (m+2^16)>>17 +// (at most 64 for any ux, so the gathers stay in the 65-entry tables), +// e = (ux>>23) - 127; +// - per four lanes, in float64 with logCR's unfused operations in its order +// (VMULPD then VADDPD, never FMA): tz from (m | 1023<<23)<<29, tr[j] and +// tl[j] gathered (VGATHERDPD), z = tz*tr - 1, r, and the bounds r and +// r + 0x1.f06p-33 rounded to float32; +// - x == 1 is 0 (logCR's early return). +// +// A block with an input that is not a positive normal float32, or a lane +// whose bounds round apart, is not stored: the kernel returns the count of +// floats it wrote, and the Go wrapper finishes that block with logCR. Needs +// AVX2 (gathers, 256-bit integer ops). +func log32CRKernel() *emit.Function { + sig := amd64.Layout( + []string{"dst", "src", "n", "k"}, + []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64, amd64.Ptr}, + []string{"done"}, []amd64.Type{amd64.Int64}, + ) + const ( + one, ln2, b0, b1, b2, w = 0, 32, 64, 96, 128, 160 + sub, maxv, mmask, jadd, bias, oneBits = 192, 224, 256, 288, 320, 352 + expOr = 384 + tr, tl = 416, 936 + ) + b := amd64.NewFunc("log32CRAVX2", sig, 0) + b.LoadArg("dst", "DI").LoadArg("src", "SI").LoadArg("n", "CX").LoadArg("k", "R8") + b.Raw(fmt.Sprintf("LEAQ %d(R8), R9", tr)) + b.Raw(fmt.Sprintf("LEAQ %d(R8), R10", tl)) + b.Raw("XORQ AX, AX") + // half runs the float64 part on the four lanes whose j is in X2, mantissa + // bits (zero-extended to 64) in Y10 and exponent (as float64) in Y11, + // leaving the bounds in ub and lb. Temporaries Y4-Y9, Y12. + half := func(ub, lb string) { + b.Raw(fmt.Sprintf("VPOR %d(R8), Y10, Y10", expOr)) + b.Raw("VPSLLQ $29, Y10, Y10") // tz, in [1,2) + b.Raw("VPCMPEQQ Y12, Y12, Y12") // gather every lane + b.Raw("VGATHERDPD Y12, (R9)(X2*8), Y4") // tr[j] + b.Raw("VPCMPEQQ Y12, Y12, Y12") + b.Raw("VGATHERDPD Y12, (R10)(X2*8), Y5") // tl[j] + b.Raw("VMULPD Y4, Y10, Y4") // tz*tr + b.Raw(fmt.Sprintf("VSUBPD %d(R8), Y4, Y4", one)) // z + b.Raw("VMULPD Y4, Y4, Y6") // z2 + b.Raw(fmt.Sprintf("VMULPD %d(R8), Y11, Y7", ln2)) // e*ln2 + b.Raw("VADDPD Y5, Y7, Y7") // + tl + b.Raw(fmt.Sprintf("VMULPD %d(R8), Y4, Y8", b0)) + b.Raw("VADDPD Y8, Y7, Y7") // + z*b0 + b.Raw(fmt.Sprintf("VMULPD %d(R8), Y4, Y8", b2)) + b.Raw(fmt.Sprintf("VADDPD %d(R8), Y8, Y8", b1)) // b1 + z*b2 + b.Raw("VMULPD Y8, Y6, Y8") + b.Raw("VADDPD Y8, Y7, Y7") // r + b.Raw(fmt.Sprintf("VADDPD %d(R8), Y7, Y9", w)) // r + 0x1.f06p-33 + b.Raw(fmt.Sprintf("VCVTPD2PSY Y7, %s", ub)) + b.Raw(fmt.Sprintf("VCVTPD2PSY Y9, %s", lb)) + } + b.Raw("lloop:") + b.Raw("VMOVDQU (SI), Y0") + // Not a positive normal: v = ux - 2^23 outside [0, 0x7effffff]. + b.Raw(fmt.Sprintf("VPSUBD %d(R8), Y0, Y4", sub)) + b.Raw(fmt.Sprintf("VPCMPGTD %d(R8), Y4, Y5", maxv)) // v > 0x7effffff + b.Raw("VPXOR Y6, Y6, Y6") + b.Raw("VPCMPGTD Y4, Y6, Y6") // 0 > v + b.Raw("VPOR Y5, Y6, Y6") + b.Raw("VPTEST Y6, Y6") + b.Raw("JNZ lbail") + b.Raw(fmt.Sprintf("VPCMPEQD %d(R8), Y0, Y13", oneBits)) // x == 1 + b.Raw(fmt.Sprintf("VPAND %d(R8), Y0, Y1", mmask)) // m + b.Raw(fmt.Sprintf("VPADDD %d(R8), Y1, Y2", jadd)) + b.Raw("VPSRLD $17, Y2, Y2") // j + b.Raw("VPSRLD $23, Y0, Y3") + b.Raw(fmt.Sprintf("VPSUBD %d(R8), Y3, Y3", bias)) // e + b.Raw("VPMOVZXDQ X1, Y10") + b.Raw("VCVTDQ2PD X3, Y11") + half("X14", "X15") + b.Raw("VEXTRACTI128 $1, Y1, X1") + b.Raw("VPMOVZXDQ X1, Y10") + b.Raw("VEXTRACTI128 $1, Y3, X3") + b.Raw("VCVTDQ2PD X3, Y11") + b.Raw("VEXTRACTI128 $1, Y2, X2") + half("X1", "X3") + b.Raw("VINSERTI128 $1, X1, Y14, Y14") // ub, eight lanes + b.Raw("VINSERTI128 $1, X3, Y15, Y15") // lb + b.Raw("VPCMPEQD Y15, Y14, Y15") // the bounds agree + b.Raw("VPOR Y13, Y15, Y15") // or x == 1 + b.Raw("VMOVMSKPS Y15, DX") + b.Raw("CMPL DX, $0xff") + b.Raw("JNE lbail") + b.Raw("VPANDN Y14, Y13, Y14") // x == 1 -> +0 + b.Raw("VMOVDQU Y14, (DI)") + b.Raw("ADDQ $32, SI").Raw("ADDQ $32, DI").Raw("ADDQ $8, AX") + b.Raw("CMPQ AX, CX").Raw("JLT lloop") + b.Raw("lbail:") + b.Raw("VZEROUPPER") + b.StoreRet("AX", "done") + b.Ret() + return b.Func() +} + // bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i], // 16 floats per iteration over four XMM registers with the packed op, then a // scalar tail with sca. Each lane is one IEEE float32 operation, so the result diff --git a/internal/kernels/expcr_other.go b/internal/kernels/expcr_other.go index ed70ad1..80b6267 100644 --- a/internal/kernels/expcr_other.go +++ b/internal/kernels/expcr_other.go @@ -5,3 +5,6 @@ package kernels // exp32Fast is the correctly rounded float32 exp: expCR in Go on these // architectures (no SIMD kernel yet). func exp32Fast(dst, src []float32) { exp32CR(dst, src) } + +// log32Fast is the correctly rounded float32 log: logCR in Go here. +func log32Fast(dst, src []float32) { logCRSlice(dst, src) } diff --git a/internal/kernels/logcr.go b/internal/kernels/logcr.go new file mode 100644 index 0000000..01e00f8 --- /dev/null +++ b/internal/kernels/logcr.go @@ -0,0 +1,138 @@ +package kernels + +import ( + "math" + "math/bits" +) + +// logCR is CORE-MATH's correctly rounded binary32 logarithm (Alexei Sibidanov +// and Paul Zimmermann, src/binary32/log/logf.c, MIT), one C operation per Go +// operation. x = 2^e·m, m in [1,2); tr[j] ≈ 1/m on 64 intervals so that +// z = m·tr[j] - 1 is small; log x = e·ln2 + tl[j] + log1p(z), with +// tl[j] = -log(tr[j]). A degree-3 polynomial gives a lower bound r of the +// exact value with r + 0x1.f06p-33 an upper bound; when both round to the +// same float32 that float32 is the answer. Otherwise (762 750 of the 2^31 +// positive finite inputs, about 1 in 2800, measured by TestLogCRMatchesLog32) +// a degree-8 polynomial with a split ln2 decides it. +// +// Every product is converted explicitly to float64, which Go defines as +// rounding it, so no architecture fuses it into an FMA: the operations are +// the C source's, unfused, everywhere. TestLogCRMatchesLog32 holds the result +// to log32, already proven correctly rounded, over every float32. +func logCR(x float32) float32 { + ux := math.Float32bits(x) + if ux < 1<<23 || ux >= 0x7f800000 { + if ux == 0 || ux >= 0x7f800000 { + return logCRSpecial(x) + } + n := bits.LeadingZeros32(ux) - 8 // subnormal: normalise the mantissa + ux <<= n + ux -= uint32(n) << 23 + } + if ux == 127<<23 { + return 0 + } + m := ux & (1<<23 - 1) + j := (m + 1<<(23-7)) >> (23 - 6) + e := float64(int32(ux)>>23 - 127) + z := float64(math.Float64frombits((uint64(m)|1023<<23)<<(52-23))*logCRTr[j]) - 1 + z2 := float64(z * z) + r := ((float64(e*0x1.62e42fefa39efp-1) + logCRTl[j]) + float64(z*logCRB[0])) + + float64(z2*(logCRB[1]+float64(z*logCRB[2]))) + ub, lb := float32(r), float32(r+0x1.f06p-33) + if ub == lb { + return ub + } + return logCRSlow(x, z, z2, e, j) +} + +// logCRSlow is logf.c's second step, for the inputs whose bounds straddle a +// float32 rounding boundary. +func logCRSlow(x float32, z, z2, e float64, j uint32) float32 { + c := &logCRC + f := float64(z2 * ((c[0] + float64(z*c[1])) + float64(z2*((c[2]+float64(z*c[3]))+ + float64(z2*(c[4]+float64(z*c[5])+float64(z2*c[6]))))))) + if math.Abs(float64(x-1)) < 0x1p-10 { + return float32(z + f) + } + f -= float64(0x1.0ca86c3898dp-49 * e) + f += z + f += logCRTl[j] - logCRTl[0] + el := float64(e * 0x1.62e42fefa3ap-1) + r := el + f + ub := float32(r) + if math.Float64bits(r)&(1<<28-1) == 0 { + dr := (el - r) + f + r += float64(dr * 64) + ub = float32(r) + } + return ub +} + +// logCRSpecial is logf.c's as_special: ±0 → -Inf, +Inf → +Inf, NaN → NaN, +// negatives → NaN. +func logCRSpecial(x float32) float32 { + switch ux := math.Float32bits(x); { + case ux<<1 == 0: + return float32(math.Inf(-1)) + case ux == 0x7f800000: + return x + case ux<<1 > 0xff000000: + return x + x + } + return float32(math.NaN()) +} + +// logCRSlice writes logCR(src[i]) into dst[i]. +func logCRSlice(dst, src []float32) { + src = src[:len(dst)] + for i, x := range src { + dst[i] = logCR(x) + } +} + +var logCRTr = [65]float64{ + 0x1p+0, 0x1.f81f82p-1, 0x1.f07c1fp-1, 0x1.e9131acp-1, + 0x1.e1e1e1ep-1, 0x1.dae6077p-1, 0x1.d41d41dp-1, 0x1.cd85689p-1, + 0x1.c71c71cp-1, 0x1.c0e0704p-1, 0x1.bacf915p-1, 0x1.b4e81b5p-1, + 0x1.af286bdp-1, 0x1.a98ef6p-1, 0x1.a41a41ap-1, 0x1.9ec8e95p-1, + 0x1.999999ap-1, 0x1.948b0fdp-1, 0x1.8f9c19p-1, 0x1.8acb90fp-1, + 0x1.8618618p-1, 0x1.8181818p-1, 0x1.7d05f41p-1, 0x1.78a4c81p-1, + 0x1.745d174p-1, 0x1.702e05cp-1, 0x1.6c16c17p-1, 0x1.6816817p-1, + 0x1.642c859p-1, 0x1.605816p-1, 0x1.5c9882cp-1, 0x1.58ed231p-1, + 0x1.5555555p-1, 0x1.51d07ebp-1, 0x1.4e5e0a7p-1, 0x1.4afd6ap-1, + 0x1.47ae148p-1, 0x1.446f865p-1, 0x1.4141414p-1, 0x1.3e22cbdp-1, + 0x1.3b13b14p-1, 0x1.3813814p-1, 0x1.3521cfbp-1, 0x1.323e34ap-1, + 0x1.2f684bep-1, 0x1.2c9fb4ep-1, 0x1.29e412ap-1, 0x1.27350b9p-1, + 0x1.2492492p-1, 0x1.21fb781p-1, 0x1.1f7047ep-1, 0x1.1cf06aep-1, + 0x1.1a7b961p-1, 0x1.1811812p-1, 0x1.15b1e5fp-1, 0x1.135c811p-1, + 0x1.1111111p-1, 0x1.0ecf56cp-1, 0x1.0c9715p-1, 0x1.0a6810ap-1, + 0x1.0842108p-1, 0x1.0624dd3p-1, 0x1.041041p-1, 0x1.0204081p-1, 0.5, +} + +var logCRTl = [65]float64{ + -0x1.3b40815cd0628p-45, 0x1.fc0a890fbb514p-7, 0x1.f829b1e780b98p-6, 0x1.77458f532c948p-5, + 0x1.f0a30c2114ef2p-5, 0x1.341d793bbc7f7p-4, 0x1.6f0d28d256172p-4, 0x1.a926d3a6acb89p-4, + 0x1.e2707722ae90cp-4, 0x1.0d77e7a90896cp-3, 0x1.29552f6fff036p-3, 0x1.44d2b6c5b7831p-3, + 0x1.5ff306ee78ee7p-3, 0x1.7ab890410d41cp-3, 0x1.9525a9e3451c7p-3, 0x1.af3c94ed0bb06p-3, + 0x1.c8ff7c59a9535p-3, 0x1.e27076d5aedf9p-3, 0x1.fb9186b5e393ep-3, 0x1.0a324e38b8e6dp-2, + 0x1.1675cacaba398p-2, 0x1.22941fc0f76efp-2, 0x1.2e8e2bc311abap-2, 0x1.3a64c56b14373p-2, + 0x1.4618bc31c5c4cp-2, 0x1.51aad874df5b7p-2, 0x1.5d1bdbea80754p-2, 0x1.686c81d331238p-2, + 0x1.739d7f6dbcd9p-2, 0x1.7eaf83c82ad4dp-2, 0x1.89a3385813fe4p-2, 0x1.947941aa91484p-2, + 0x1.9f323edbf95d5p-2, 0x1.a9cec9a4205d3p-2, 0x1.b44f77c5c8cecp-2, 0x1.beb4d9ea71905p-2, + 0x1.c8ff7c69a97abp-2, 0x1.d32fe7f38e95fp-2, 0x1.dd46a0501c22ap-2, 0x1.e7442617e8511p-2, + 0x1.f128f5eaf0476p-2, 0x1.faf588dd8f0a8p-2, 0x1.02552a5edcfc4p-1, 0x1.0723e5c64de05p-1, + 0x1.0be72e3852947p-1, 0x1.109f39d554b5cp-1, 0x1.154c3d2c4d4aep-1, 0x1.19ee6b38bc834p-1, + 0x1.1e85f5ef03f95p-1, 0x1.23130d7fabe07p-1, 0x1.2795e1219afep-1, 0x1.2c0e9ec9c8d5p-1, + 0x1.307d7337f0f83p-1, 0x1.34e289cb4e098p-1, 0x1.393e0d42e28dep-1, 0x1.3d9026ad555bfp-1, + 0x1.41d8fe8667173p-1, 0x1.4618bc1ec5d87p-1, 0x1.4a4f85d303d8p-1, 0x1.4e7d8127f5a75p-1, + 0x1.52a2d26dbc47p-1, 0x1.56bf9d597f25ep-1, 0x1.5ad404cb59df2p-1, 0x1.5ee02a928153ap-1, + 0x1.62e42fefa38b4p-1, +} + +var logCRB = [3]float64{0x1.00000006342eap+0, -0x1.0001f7fdc3977p-1, 0x1.554a4e5cae9cfp-2} + +var logCRC = [7]float64{ + -0x1p-1, 0x1.55555555571cap-2, -0x1.0000000002d85p-2, 0x1.9999987d0c963p-3, + -0x1.555554059a8bbp-3, 0x1.24aebcf71a38fp-3, -0x1.001c73915d758p-3, +} diff --git a/internal/kernels/logcr_amd64.go b/internal/kernels/logcr_amd64.go new file mode 100644 index 0000000..e0d005b --- /dev/null +++ b/internal/kernels/logcr_amd64.go @@ -0,0 +1,52 @@ +package kernels + +// logCRConsts is the constant block log32CRAVX2 reads (asmgen/amd64/gen.go, +// log32CRKernel), at the offsets the kernel uses. +type logCRConsts struct { + one, ln2, b0, b1, b2, w [4]float64 // offsets 0, 32, 64, 96, 128, 160 + sub, maxv, mmask, jadd, bias, oneBits [8]uint32 // 192, 224, 256, 288, 320, 352 + expOr [4]uint64 // 384 + tr, tl [65]float64 // 416, 936 +} + +var logCRK = func() *logCRConsts { + k := new(logCRConsts) + for i := range 4 { + k.one[i], k.ln2[i], k.w[i] = 1, 0x1.62e42fefa39efp-1, 0x1.f06p-33 + k.b0[i], k.b1[i], k.b2[i] = logCRB[0], logCRB[1], logCRB[2] + k.expOr[i] = 1023 << 23 + } + for i := range 8 { + k.sub[i], k.maxv[i], k.mmask[i] = 1<<23, 0x7effffff, 1<<23-1 + k.jadd[i], k.bias[i], k.oneBits[i] = 1<<16, 127, 0x3f800000 + } + k.tr, k.tl = logCRTr, logCRTl + return k +}() + +// useLog32AVX2 selects the AVX2 kernel; a var so tests can force the Go path. +var useLog32AVX2 = hasAVX2() + +// log32Fast is the correctly rounded float32 log: blocks of 8 on the AVX2 +// kernel, which hands back each block it cannot finish (an input that is not +// a positive normal float32, or bounds that round apart, about 1 lane in +// 2800) to logCR, as is the tail. Both run logCR's operations in its order, +// so they agree bit for bit. +func log32Fast(dst, src []float32) { + src = src[:len(dst)] + i := 0 + if useLog32AVX2 { + n := len(dst) &^ 7 + for i < n { + i += log32CRAVX2(&dst[i], &src[i], n-i, logCRK) + if i < n { + logCRSlice(dst[i:i+8], src[i:i+8]) + i += 8 + } + } + } + logCRSlice(dst[i:], src[i:]) +} + +//go:noescape +func log32CRAVX2(dst, src *float32, n int, k *logCRConsts) (done int) diff --git a/internal/kernels/logcr_amd64_test.go b/internal/kernels/logcr_amd64_test.go new file mode 100644 index 0000000..624f445 --- /dev/null +++ b/internal/kernels/logcr_amd64_test.go @@ -0,0 +1,91 @@ +package kernels + +import ( + "math" + "math/rand" + "testing" + "unsafe" +) + +// TestLogCRConstsLayout holds logCRConsts to the offsets log32CRKernel uses. +func TestLogCRConstsLayout(t *testing.T) { + var k logCRConsts + for _, c := range []struct { + name string + got, want uintptr + }{ + {"one", unsafe.Offsetof(k.one), 0}, {"ln2", unsafe.Offsetof(k.ln2), 32}, + {"b0", unsafe.Offsetof(k.b0), 64}, {"b1", unsafe.Offsetof(k.b1), 96}, + {"b2", unsafe.Offsetof(k.b2), 128}, {"w", unsafe.Offsetof(k.w), 160}, + {"sub", unsafe.Offsetof(k.sub), 192}, {"maxv", unsafe.Offsetof(k.maxv), 224}, + {"mmask", unsafe.Offsetof(k.mmask), 256}, {"jadd", unsafe.Offsetof(k.jadd), 288}, + {"bias", unsafe.Offsetof(k.bias), 320}, {"oneBits", unsafe.Offsetof(k.oneBits), 352}, + {"expOr", unsafe.Offsetof(k.expOr), 384}, {"tr", unsafe.Offsetof(k.tr), 416}, + {"tl", unsafe.Offsetof(k.tl), 936}, + } { + if c.got != c.want { + t.Errorf("%s at %d, the kernel reads %d", c.name, c.got, c.want) + } + } +} + +// TestLog32AVX2MatchesGo holds log32Fast (AVX2 blocks, Go for the blocks it +// hands back and the tail) to logCR bit for bit (NaN to NaN): random bit +// patterns (a third of them negative, NaN or subnormal, so most blocks bail), +// random positive normals (few bail), 1.0 runs, the special values, and every +// length 0..70. +func TestLog32AVX2MatchesGo(t *testing.T) { + if !useLog32AVX2 { + t.Skip("no AVX2 here") + } + r := rand.New(rand.NewSource(5)) + src := make([]float32, 0, 300000) + for _, v := range []float32{1, 1, 1, 1, 1, 1, 1, 1, 0, float32(math.Copysign(0, -1)), -1, + float32(math.Inf(1)), float32(math.Inf(-1)), float32(math.NaN()), math.SmallestNonzeroFloat32, + 0x1p-126, math.MaxFloat32, 0.99999994, 1.0000001, 2, 0.5} { + src = append(src, v) + } + for len(src) < 100000 { + src = append(src, math.Float32frombits(r.Uint32())) + } + for len(src) < cap(src) { + src = append(src, math.Float32frombits(0x00800000+uint32(r.Int63n(0x7f000000)))) + } + want := make([]float32, len(src)) + logCRSlice(want, src) + got := make([]float32, len(src)) + log32Fast(got, src) + same := func(a, b float32) bool { return math.Float32bits(a) == math.Float32bits(b) || (a != a && b != b) } + for i := range src { + if !same(got[i], want[i]) { + t.Fatalf("log(0x%08x): AVX2 0x%08x, Go 0x%08x", math.Float32bits(src[i]), math.Float32bits(got[i]), math.Float32bits(want[i])) + } + } + for n := 0; n <= 70; n++ { + d := make([]float32, n) + log32Fast(d, src[:n]) + for i := range d { + if !same(d[i], want[i]) { + t.Fatalf("length %d [%d]: 0x%08x, want 0x%08x", n, i, math.Float32bits(d[i]), math.Float32bits(want[i])) + } + } + } +} + +// TestLog32AVX2InPlace runs the kernel with dst == src: a block it hands +// back must still hold its inputs. +func TestLog32AVX2InPlace(t *testing.T) { + r := rand.New(rand.NewSource(6)) + src := make([]float32, 4099) + for i := range src { + src[i] = math.Float32frombits(r.Uint32() &^ (1 << 31)) // NaN, Inf and subnormals among them + } + want := make([]float32, len(src)) + logCRSlice(want, src) + log32Fast(src, src) + for i := range src { + if math.Float32bits(src[i]) != math.Float32bits(want[i]) && !(src[i] != src[i] && want[i] != want[i]) { + t.Fatalf("[%d]: 0x%08x, want 0x%08x", i, math.Float32bits(src[i]), math.Float32bits(want[i])) + } + } +} diff --git a/internal/kernels/logcr_fence_amd64_test.go b/internal/kernels/logcr_fence_amd64_test.go new file mode 100644 index 0000000..26c1604 --- /dev/null +++ b/internal/kernels/logcr_fence_amd64_test.go @@ -0,0 +1,20 @@ +//go:build linux || darwin + +package kernels + +import "testing" + +// TestLog32AVX2StaysInside fences src and dst of the AVX2 log kernel +// (fences32 needs mmap/mprotect, hence the build tag of fence_test.go). +func TestLog32AVX2StaysInside(t *testing.T) { + if !useLog32AVX2 { + t.Skip("no AVX2 here") + } + for n := 8; n <= 64; n += 8 { + for _, s := range fences32(t, n) { + for _, d := range fences32(t, n) { + log32CRAVX2(&d[0], &s[0], n, logCRK) + } + } + } +} diff --git a/internal/kernels/logcr_test.go b/internal/kernels/logcr_test.go new file mode 100644 index 0000000..4b38e0f --- /dev/null +++ b/internal/kernels/logcr_test.go @@ -0,0 +1,127 @@ +package kernels + +import ( + "math" + "math/bits" + "runtime" + "sync" + "sync/atomic" + "testing" +) + +// TestLogCRMatchesLog32 holds logCR to log32 bit for bit (NaN to NaN) on +// every float32 input (NDARRAY_EXHAUSTIVE=1; otherwise every 65537th). +// log32 is proven correctly rounded over all 2^32 inputs by +// TestLog32CorrectlyRounded, so equality everywhere proves logCR is too. It +// also counts the inputs logCRSlow decides. +func TestLogCRMatchesLog32(t *testing.T) { + stride := exhaustiveStride() + const batch = 4096 + var next atomic.Uint64 + var bad, slow, checked atomic.Int64 + var mu sync.Mutex + var first []string + var wg sync.WaitGroup + for range runtime.GOMAXPROCS(0) { + wg.Add(1) + go func() { + defer wg.Done() + xs := make([]float32, 0, batch) + want := make([]float32, batch) + for { + start := next.Add(batch*stride) - batch*stride + if start >= 1<<32 { + return + } + xs = xs[:0] + for b := start; b < start+batch*stride && b < 1<<32; b += stride { + xs = append(xs, math.Float32frombits(uint32(b))) + } + log32(want[:len(xs)], xs) + for i, x := range xs { + checked.Add(1) + got := logCR(x) + if logCRTakesSlow(x) { + slow.Add(1) + } + w := want[i] + if math.Float32bits(got) != math.Float32bits(w) && !(got != got && w != w) { + bad.Add(1) + mu.Lock() + if len(first) < 8 { + first = append(first, "logCR("+formatF32(x)+") = "+formatF32(got)+", log32 "+formatF32(w)) + } + mu.Unlock() + } + } + } + }() + } + wg.Wait() + t.Logf("%d inputs checked, %d through the slow path, %d differ from log32", checked.Load(), slow.Load(), bad.Load()) + for _, s := range first { + t.Error(s) + } +} + +// logCRTakesSlow repeats logCR's fast path and reports whether its bounds +// straddle a rounding boundary. +func logCRTakesSlow(x float32) bool { + ux := math.Float32bits(x) + if ux == 0 || ux >= 0x7f800000 || ux == 127<<23 { + return false + } + if ux < 1<<23 { + n := bits.LeadingZeros32(ux) - 8 + ux <<= n + ux -= uint32(n) << 23 + } + m := ux & (1<<23 - 1) + j := (m + 1<<(23-7)) >> (23 - 6) + e := float64(int32(ux)>>23 - 127) + z := float64(math.Float64frombits((uint64(m)|1023<<23)<<(52-23))*logCRTr[j]) - 1 + z2 := float64(z * z) + r := ((float64(e*0x1.62e42fefa39efp-1) + logCRTl[j]) + float64(z*logCRB[0])) + + float64(z2*(logCRB[1]+float64(z*logCRB[2]))) + return float32(r) != float32(r+0x1.f06p-33) +} + +// BenchmarkLog32Kernels compares the float32 log kernels on 16 Ki inputs in +// [0.25, 24.25), the range of benchmarks/BenchmarkDTypeExpLog. +func BenchmarkLog32Kernels(b *testing.B) { + src := make([]float32, 16<<10) + for i := range src { + src[i] = 0.25 + 24*float32(i)/float32(len(src)) + } + dst := make([]float32, len(src)) + for _, k := range []struct { + name string + f func(dst, src []float32) + }{{"via64", log32}, {"logCR", logCRSlice}} { + b.Run(k.name, func(b *testing.B) { + b.SetBytes(int64(4 * len(src))) + for b.Loop() { + k.f(dst, src) + } + }) + } +} + +func TestLogCRSpecial(t *testing.T) { + for _, c := range []struct{ x, want float32 }{ + {0, float32(math.Inf(-1))}, + {float32(math.Copysign(0, -1)), float32(math.Inf(-1))}, + {float32(math.Inf(1)), float32(math.Inf(1))}, + {1, 0}, + {math.SmallestNonzeroFloat32, -103.27893}, + } { + if got := logCR(c.x); got != c.want { + t.Errorf("logCR(%v) = %v, want %v", c.x, got, c.want) + } + } + for _, x := range []float32{-1, float32(math.Inf(-1)), float32(math.NaN()), -math.SmallestNonzeroFloat32} { + if got := logCR(x); got == got { + t.Errorf("logCR(%v) = %v, want NaN", x, got) + } + } +} diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index de6f8f0..0a86822 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -1781,6 +1781,95 @@ TEXT ·exp32CRAVX2(SB), NOSPLIT, $0-32 VZEROUPPER RET +TEXT ·log32CRAVX2(SB), NOSPLIT, $0-40 + MOVQ dst+0(FP), DI + MOVQ src+8(FP), SI + MOVQ n+16(FP), CX + MOVQ k+24(FP), R8 + LEAQ 416(R8), R9 + LEAQ 936(R8), R10 + XORQ AX, AX + lloop: + VMOVDQU (SI), Y0 + VPSUBD 192(R8), Y0, Y4 + VPCMPGTD 224(R8), Y4, Y5 + VPXOR Y6, Y6, Y6 + VPCMPGTD Y4, Y6, Y6 + VPOR Y5, Y6, Y6 + VPTEST Y6, Y6 + JNZ lbail + VPCMPEQD 352(R8), Y0, Y13 + VPAND 256(R8), Y0, Y1 + VPADDD 288(R8), Y1, Y2 + VPSRLD $17, Y2, Y2 + VPSRLD $23, Y0, Y3 + VPSUBD 320(R8), Y3, Y3 + VPMOVZXDQ X1, Y10 + VCVTDQ2PD X3, Y11 + VPOR 384(R8), Y10, Y10 + VPSLLQ $29, Y10, Y10 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R9)(X2*8), Y4 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R10)(X2*8), Y5 + VMULPD Y4, Y10, Y4 + VSUBPD 0(R8), Y4, Y4 + VMULPD Y4, Y4, Y6 + VMULPD 32(R8), Y11, Y7 + VADDPD Y5, Y7, Y7 + VMULPD 64(R8), Y4, Y8 + VADDPD Y8, Y7, Y7 + VMULPD 128(R8), Y4, Y8 + VADDPD 96(R8), Y8, Y8 + VMULPD Y8, Y6, Y8 + VADDPD Y8, Y7, Y7 + VADDPD 160(R8), Y7, Y9 + VCVTPD2PSY Y7, X14 + VCVTPD2PSY Y9, X15 + VEXTRACTI128 $1, Y1, X1 + VPMOVZXDQ X1, Y10 + VEXTRACTI128 $1, Y3, X3 + VCVTDQ2PD X3, Y11 + VEXTRACTI128 $1, Y2, X2 + VPOR 384(R8), Y10, Y10 + VPSLLQ $29, Y10, Y10 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R9)(X2*8), Y4 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R10)(X2*8), Y5 + VMULPD Y4, Y10, Y4 + VSUBPD 0(R8), Y4, Y4 + VMULPD Y4, Y4, Y6 + VMULPD 32(R8), Y11, Y7 + VADDPD Y5, Y7, Y7 + VMULPD 64(R8), Y4, Y8 + VADDPD Y8, Y7, Y7 + VMULPD 128(R8), Y4, Y8 + VADDPD 96(R8), Y8, Y8 + VMULPD Y8, Y6, Y8 + VADDPD Y8, Y7, Y7 + VADDPD 160(R8), Y7, Y9 + VCVTPD2PSY Y7, X1 + VCVTPD2PSY Y9, X3 + VINSERTI128 $1, X1, Y14, Y14 + VINSERTI128 $1, X3, Y15, Y15 + VPCMPEQD Y15, Y14, Y15 + VPOR Y13, Y15, Y15 + VMOVMSKPS Y15, DX + CMPL DX, $0xff + JNE lbail + VPANDN Y14, Y13, Y14 + VMOVDQU Y14, (DI) + ADDQ $32, SI + ADDQ $32, DI + ADDQ $8, AX + CMPQ AX, CX + JLT lloop + lbail: + VZEROUPPER + MOVQ AX, done+32(FP) + RET + TEXT ·blocks8SSE2(SB), NOSPLIT, $0-24 MOVQ r+0(FP), DI MOVQ a+8(FP), SI From 92c2925baf1f5561668f04fee04fb833e55d49ef Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 2 Nov 2026 21:51:59 +0100 Subject: [PATCH 2/2] float32 Log: route through logCR and its AVX2 kernel Log32 and Log32P now run log32Fast: log32CRAVX2 on amd64 with AVX2, logCR in Go elsewhere. TestLog32CorrectlyRounded, run on all 2^32 inputs against the 300-bit reference, finds 0 incorrectly rounded on darwin/arm64 (Go) and Zen 3 (AVX2). The via64 log32 and its 24-entry exception table move to a test file as the independent oracle. perf.md, README and SECURITY.md describe the new path; float32 Log goes from 3x NumPy to parity on one Zen 3 core. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- SECURITY.md | 11 ++++ docs/perf.md | 52 +++++++++++++++-- internal/kernels/logcr_test.go | 2 +- internal/kernels/via64.go | 81 +++------------------------ internal/kernels/via64_oracle_test.go | 68 ++++++++++++++++++++++ 6 files changed, 136 insertions(+), 81 deletions(-) create mode 100644 internal/kernels/via64_oracle_test.go diff --git a/README.md b/README.md index 03ea6d6..20de4f3 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ amd64 with FMA (AVX2), ppc64le (VSX) and s390x (z14 vector), accumulating in float32 like NumPy's sgemm, and elsewhere on the float64 GEMM (accumulated in float64, rounded once). Its `Exp` and `Log` are **correctly rounded**, proven by an exhaustive test - over all 2³² float32 inputs + over all 2³² float32 inputs, and on amd64 with AVX2 they run as fast as + NumPy's float32 SIMD, which is not correctly rounded ([docs/design-dtypes.md](docs/design-dtypes.md)). - **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/ `Eye`/`Identity`. diff --git a/SECURITY.md b/SECURITY.md index 020f9d4..3e01603 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,17 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-11-02 (v0.34.0).** The AVX2 float32 log kernel (`log32CRAVX2`) reads +and writes at most `n` floats, a positive multiple of 8 chosen by the Go +wrapper, and returns how many it wrote, a multiple of 8 no larger than `n`; +the wrapper finishes the block it stopped at in Go and calls it again past +that block, so the loop always advances. Its gather index is +`(m + 2^16) >> 17` with `m` the 23 mantissa bits, at most 64 for any input, +so it cannot leave the 65-entry tables. A block it stops at is not written, +so `dst == src` is safe (`TestLog32AVX2InPlace`). It is gated on AVX2 (CPU +and OS) and fenced with PROT_NONE on both sides of each operand. The Go +`logCR` indexes the same tables with the same bound. + **2026-11-02 (v0.33.0).** The AVX2 float32 exp kernel (`exp32CRAVX2`) reads and writes `n` floats, a positive multiple of 8 chosen by the Go wrapper. It gathers table entries at `z & 63`, so the index cannot leave the diff --git a/docs/perf.md b/docs/perf.md index 3e25f1c..776106b 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1515,11 +1515,53 @@ These are +235% and +289% in throughput (`abbench`, 8 rounds, p = 0.000). The host was at load 59 on 128 cores; NumPy ran in the same session on the same pinned core. -**Open defect: float32 `Log` is still 3× slower than NumPy** (`simd_log_f32`, -3.83 ULP in its source). The next step is the same treatment for log, and -the other architectures for exp. float64 `Log` is also 1.23× slower than -NumPy at 16 Ki on Zen 3 (85 µs against 69 µs), while float64 `Exp` is 16% -faster. +#### float32 `Log` in AVX2, correctly rounded and as fast as NumPy (2026-11-02) + +float32 `Log` was 3× slower than NumPy (`simd_log_f32`, 3.83 ULP in its +source). optimized-routines has no correctly rounded logf; CORE-MATH has one +(`src/binary32/log/logf.c`, Alexei Sibidanov and Paul Zimmermann, MIT). + +- **`logCR`** transcribes it. A 64-entry table and a degree-3 polynomial in + float64 give a lower bound r of log x, with r + 0x1.f06p-33 an upper + bound; when both round to the same float32, that is the answer. Otherwise + a degree-8 polynomial decides: 762 750 of the 2³¹ positive inputs, about + 1 in 2800. Every product is converted to float64 explicitly, which Go + defines as rounding it, so no architecture fuses it into an FMA. +- **Proof, two ways, over all 2³² inputs:** `logCR` equals v0.33.0's `log32` + (widened float64 `Log` plus a 24-entry exception table, no shared code) on + every input, and `Log32` is held to the 300-bit reference by + `TestLog32CorrectlyRounded`: **0 incorrectly rounded**, on darwin/arm64 + (Go) and on Zen 3 (AVX2). +- **`log32CRAVX2`** (go-asmgen, amd64 with AVX2) runs the fast path on eight + floats at a time. The integer part is done in 32-bit lanes, the tables are + gathered with `VGATHERDPD`, and the arithmetic is `VMULPD`/`VADDPD` in + `logCR`'s order, never fused. A block holding a lane that is not a positive + normal float32, or whose bounds round apart, is not stored; it is handed + back to `logCR`. x = 1 gives +0 in the vector. The kernel is bit-identical + to `logCR` on 300 000 inputs and every length 0–70, in place too, and it + is fenced. +- The old via64 `log32` and its exception table now live in a test file as + the oracle; the 24 hardest inputs stay a regression test. + +| float32 `Log`, 16 Ki in [0.25, 24.25), one Zen 3 core, load 0.35 | time | +|---|---:| +| v0.33.0 (via64) | 91.5 µs | +| `logCR` in Go | 82 µs | +| **`log32CRAVX2`** | **31.1 µs** | +| NumPy 2.5.4, `np.log(a, out=o)` | 31.0 µs | + +Through the API (`BenchmarkDTypeExpLog`, allocating), against v0.33.0, +`abbench` 8 interleaved rounds with load at most 8: −65% at 16 Ki (104.6 → +37.1 µs) and −68% at 1 Mi (6.74 → 2.16 ms), p = 0.000. + +The four gathers per block are not the bottleneck: replacing the `tl` gather +by a register copy saves at most 8%. Doing better than NumPy needs a +different algorithm, not a tweak of this one. + +**Open defect:** float64 `Log` is 1.23× slower than NumPy at 16 Ki on Zen 3 +(85 µs against 69 µs), while float64 `Exp` is 16% faster. The float32 kernels +for the other architectures are still to write; there `Exp` and `Log` run +the Go transcriptions. ## SIMD coverage diff --git a/internal/kernels/logcr_test.go b/internal/kernels/logcr_test.go index 4b38e0f..489571f 100644 --- a/internal/kernels/logcr_test.go +++ b/internal/kernels/logcr_test.go @@ -97,7 +97,7 @@ func BenchmarkLog32Kernels(b *testing.B) { for _, k := range []struct { name string f func(dst, src []float32) - }{{"via64", log32}, {"logCR", logCRSlice}} { + }{{"via64", log32}, {"logCR", logCRSlice}, {"fast", log32Fast}} { b.Run(k.name, func(b *testing.B) { b.SetBytes(int64(4 * len(src))) for b.Loop() { diff --git a/internal/kernels/via64.go b/internal/kernels/via64.go index 3ef320f..615ca01 100644 --- a/internal/kernels/via64.go +++ b/internal/kernels/via64.go @@ -1,98 +1,31 @@ package kernels -import "math" - // Correctly rounded float32 Exp and Log. // // float32 Exp and Log went through math.Exp and math.Log per element, 6-14x // slower than NumPy's float32 SIMD on one Zen 3 core. Both now compute in // float64 and round once, and both are proven correctly rounded over every -// float32 input (via64_exhaustive_test.go, expcr_test.go): +// float32 input (via64_exhaustive_test.go, expcr_test.go, logcr_test.go): // -// - Exp runs expCR (expcr.go), Arm's correctly rounded expf algorithm, on -// an AVX2 kernel where there is one (expcr_amd64.go), in Go elsewhere; -// - Log widens to this package's float64 Log (0.504 ULP) through a -// 256-element stack buffer, rounds once, and takes the 24 inputs whose -// float64 result lies next to a float32 rounding midpoint from a table. +// - Exp runs expCR (expcr.go), Arm's correctly rounded expf algorithm; +// - Log runs logCR (logcr.go), CORE-MATH's correctly rounded logf; // -// Nothing is allocated. Log is called directly, not through a func value, -// which would make the buffer escape. +// each on an AVX2 kernel where there is one (expcr_amd64.go, logcr_amd64.go), +// in Go elsewhere. Nothing is allocated. const via64Chunk = 256 -// log32 writes the correctly rounded log(src[i]) into dst[i]. -func log32(dst, src []float32) { - var buf [via64Chunk]float64 - src = src[:len(dst)] - for i := 0; i < len(dst); i += via64Chunk { - b := buf[:min(via64Chunk, len(dst)-i)] - s := src[i : i+len(b)] - for j := range b { - b[j] = float64(s[j]) - } - Log(b, b) - d := dst[i : i+len(b)] - for j, v := range b { - f := float32(v) - // Within 8 float64 ULP of a float32 rounding midpoint (the 29 - // bits float32 drops are 0x10000000 +- 8): the double rounding - // may be wrong, so take the table's answer. - if math.Float64bits(v)&(1<<29-1)-(1<<28-8) < 16 { - f = log32Exception(s[j], f) - } - d[j] = f - } - } -} - -// log32Exceptions are the float32 inputs whose float64 log lies within 2 -// float64 ULP of a float32 rounding midpoint, sorted, each with its correctly -// rounded log decided by a 300-bit reference. The exhaustive test lists them -// (NDARRAY_EXHAUSTIVE=1 NDARRAY_DUMP_HARD=1, TestLog32CorrectlyRounded): 24 -// inputs, 5 of which rounded to the wrong float32 without this table. Log's -// float64 arithmetic is the same on every architecture (its multiply-add is -// math.FMA), so the list is too. -var log32Exceptions = [...][2]uint32{ - {0x0dc8bba4, 0xc289bac4}, {0x111c87f8, 0xc28085df}, {0x1a8446cb, 0xc24d0a45}, - {0x1f116ab8, 0xc233b53e}, {0x28e3fa26, 0xc1fa75e0}, {0x29e6126b, 0xc1ef4c02}, - {0x29fd22f8, 0xc1ee8859}, {0x2c4c24b7, 0xc1d48710}, {0x2e492984, 0xc1be76ee}, - {0x38dcbe38, 0xc1128ba4}, {0x3bf86ef0, 0xc09c399e}, {0x3c413d3a, 0xc08e158f}, - {0x41178feb, 0x400fe5e7}, {0x464d5b2b, 0x4117bcf1}, {0x4665a9a6, 0x41198725}, - {0x4c5d65a5, 0x418f034b}, {0x4d604ebe, 0x419a352c}, {0x4e85f412, 0x41a6b811}, - {0x5ee8984e, 0x422e4a21}, {0x65d890d3, 0x4254d1f9}, {0x66a8c860, 0x42595e46}, - {0x66abbd63, 0x4259700e}, {0x6f31a8ec, 0x42845a89}, {0x79e7ec37, 0x42a1ffb7}, -} - -// log32Exception returns the table's correctly rounded log of x if x is in -// log32Exceptions, and f otherwise. -func log32Exception(x, f float32) float32 { - bits := math.Float32bits(x) - lo, hi := 0, len(log32Exceptions) - for lo < hi { - m := (lo + hi) / 2 - switch e := log32Exceptions[m]; { - case e[0] == bits: - return math.Float32frombits(e[1]) - case e[0] < bits: - lo = m + 1 - default: - hi = m - } - } - return f -} - // Exp32 writes the correctly rounded exp(src[i]) into dst[i]. func Exp32(dst, src []float32) { exp32Fast(dst, src) } // Log32 writes the correctly rounded log(src[i]) into dst[i]. -func Log32(dst, src []float32) { log32(dst, src) } +func Log32(dst, src []float32) { log32Fast(dst, src) } // Exp32P is Exp32, split across cores above ParThreshold. func Exp32P(dst, src []float32) { split32(exp32Fast, dst, src) } // Log32P is Log32, split across cores above ParThreshold. -func Log32P(dst, src []float32) { split32(log32, dst, src) } +func Log32P(dst, src []float32) { split32(log32Fast, dst, src) } func split32(f func(dst, src []float32), dst, src []float32) { n := len(dst) diff --git a/internal/kernels/via64_oracle_test.go b/internal/kernels/via64_oracle_test.go new file mode 100644 index 0000000..659a324 --- /dev/null +++ b/internal/kernels/via64_oracle_test.go @@ -0,0 +1,68 @@ +package kernels + +import "math" + +// log32 is the float32 log of v0.32.0 and v0.33.0, kept as an independent +// oracle for logCR (TestLogCRMatchesLog32): it widens to this package's +// float64 Log (0.504 ULP), rounds once, and takes the 24 inputs whose float64 +// result lies next to a float32 rounding midpoint from a table. It shares no +// code or constant with logCR. +func log32(dst, src []float32) { + var buf [via64Chunk]float64 + src = src[:len(dst)] + for i := 0; i < len(dst); i += via64Chunk { + b := buf[:min(via64Chunk, len(dst)-i)] + s := src[i : i+len(b)] + for j := range b { + b[j] = float64(s[j]) + } + Log(b, b) + d := dst[i : i+len(b)] + for j, v := range b { + f := float32(v) + // Within 8 float64 ULP of a float32 rounding midpoint (the 29 + // bits float32 drops are 0x10000000 +- 8): the double rounding + // may be wrong, so take the table's answer. + if math.Float64bits(v)&(1<<29-1)-(1<<28-8) < 16 { + f = log32Exception(s[j], f) + } + d[j] = f + } + } +} + +// log32Exceptions are the float32 inputs whose float64 log lies within 2 +// float64 ULP of a float32 rounding midpoint, sorted, each with its correctly +// rounded log decided by a 300-bit reference (NDARRAY_EXHAUSTIVE=1 +// NDARRAY_DUMP_HARD=1 on the v0.32.0 TestLog32CorrectlyRounded): 24 inputs, 5 +// of which rounded to the wrong float32 without this table. They stay the +// hardest known inputs for any float32 log (TestLog32Exceptions). +var log32Exceptions = [...][2]uint32{ + {0x0dc8bba4, 0xc289bac4}, {0x111c87f8, 0xc28085df}, {0x1a8446cb, 0xc24d0a45}, + {0x1f116ab8, 0xc233b53e}, {0x28e3fa26, 0xc1fa75e0}, {0x29e6126b, 0xc1ef4c02}, + {0x29fd22f8, 0xc1ee8859}, {0x2c4c24b7, 0xc1d48710}, {0x2e492984, 0xc1be76ee}, + {0x38dcbe38, 0xc1128ba4}, {0x3bf86ef0, 0xc09c399e}, {0x3c413d3a, 0xc08e158f}, + {0x41178feb, 0x400fe5e7}, {0x464d5b2b, 0x4117bcf1}, {0x4665a9a6, 0x41198725}, + {0x4c5d65a5, 0x418f034b}, {0x4d604ebe, 0x419a352c}, {0x4e85f412, 0x41a6b811}, + {0x5ee8984e, 0x422e4a21}, {0x65d890d3, 0x4254d1f9}, {0x66a8c860, 0x42595e46}, + {0x66abbd63, 0x4259700e}, {0x6f31a8ec, 0x42845a89}, {0x79e7ec37, 0x42a1ffb7}, +} + +// log32Exception returns the table's correctly rounded log of x if x is in +// log32Exceptions, and f otherwise. +func log32Exception(x, f float32) float32 { + bits := math.Float32bits(x) + lo, hi := 0, len(log32Exceptions) + for lo < hi { + m := (lo + hi) / 2 + switch e := log32Exceptions[m]; { + case e[0] == bits: + return math.Float32frombits(e[1]) + case e[0] < bits: + lo = m + 1 + default: + hi = m + } + } + return f +}