diff --git a/README.md b/README.md index 03ea6d6..20de4f3 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ amd64 with FMA (AVX2), ppc64le (VSX) and s390x (z14 vector), accumulating in float32 like NumPy's sgemm, and elsewhere on the float64 GEMM (accumulated in float64, rounded once). Its `Exp` and `Log` are **correctly rounded**, proven by an exhaustive test - over all 2³² float32 inputs + over all 2³² float32 inputs, and on amd64 with AVX2 they run as fast as + NumPy's float32 SIMD, which is not correctly rounded ([docs/design-dtypes.md](docs/design-dtypes.md)). - **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/ `Eye`/`Identity`. diff --git a/SECURITY.md b/SECURITY.md index 020f9d4..3e01603 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,17 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-11-02 (v0.34.0).** The AVX2 float32 log kernel (`log32CRAVX2`) reads +and writes at most `n` floats, a positive multiple of 8 chosen by the Go +wrapper, and returns how many it wrote, a multiple of 8 no larger than `n`; +the wrapper finishes the block it stopped at in Go and calls it again past +that block, so the loop always advances. Its gather index is +`(m + 2^16) >> 17` with `m` the 23 mantissa bits, at most 64 for any input, +so it cannot leave the 65-entry tables. A block it stops at is not written, +so `dst == src` is safe (`TestLog32AVX2InPlace`). It is gated on AVX2 (CPU +and OS) and fenced with PROT_NONE on both sides of each operand. The Go +`logCR` indexes the same tables with the same bound. + **2026-11-02 (v0.33.0).** The AVX2 float32 exp kernel (`exp32CRAVX2`) reads and writes `n` floats, a positive multiple of 8 chosen by the Go wrapper. It gathers table entries at `z & 63`, so the index cannot leave the diff --git a/docs/perf.md b/docs/perf.md index 3e25f1c..776106b 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1515,11 +1515,53 @@ These are +235% and +289% in throughput (`abbench`, 8 rounds, p = 0.000). The host was at load 59 on 128 cores; NumPy ran in the same session on the same pinned core. -**Open defect: float32 `Log` is still 3× slower than NumPy** (`simd_log_f32`, -3.83 ULP in its source). The next step is the same treatment for log, and -the other architectures for exp. float64 `Log` is also 1.23× slower than -NumPy at 16 Ki on Zen 3 (85 µs against 69 µs), while float64 `Exp` is 16% -faster. +#### float32 `Log` in AVX2, correctly rounded and as fast as NumPy (2026-11-02) + +float32 `Log` was 3× slower than NumPy (`simd_log_f32`, 3.83 ULP in its +source). optimized-routines has no correctly rounded logf; CORE-MATH has one +(`src/binary32/log/logf.c`, Alexei Sibidanov and Paul Zimmermann, MIT). + +- **`logCR`** transcribes it. A 64-entry table and a degree-3 polynomial in + float64 give a lower bound r of log x, with r + 0x1.f06p-33 an upper + bound; when both round to the same float32, that is the answer. Otherwise + a degree-8 polynomial decides: 762 750 of the 2³¹ positive inputs, about + 1 in 2800. Every product is converted to float64 explicitly, which Go + defines as rounding it, so no architecture fuses it into an FMA. +- **Proof, two ways, over all 2³² inputs:** `logCR` equals v0.33.0's `log32` + (widened float64 `Log` plus a 24-entry exception table, no shared code) on + every input, and `Log32` is held to the 300-bit reference by + `TestLog32CorrectlyRounded`: **0 incorrectly rounded**, on darwin/arm64 + (Go) and on Zen 3 (AVX2). +- **`log32CRAVX2`** (go-asmgen, amd64 with AVX2) runs the fast path on eight + floats at a time. The integer part is done in 32-bit lanes, the tables are + gathered with `VGATHERDPD`, and the arithmetic is `VMULPD`/`VADDPD` in + `logCR`'s order, never fused. A block holding a lane that is not a positive + normal float32, or whose bounds round apart, is not stored; it is handed + back to `logCR`. x = 1 gives +0 in the vector. The kernel is bit-identical + to `logCR` on 300 000 inputs and every length 0–70, in place too, and it + is fenced. +- The old via64 `log32` and its exception table now live in a test file as + the oracle; the 24 hardest inputs stay a regression test. + +| float32 `Log`, 16 Ki in [0.25, 24.25), one Zen 3 core, load 0.35 | time | +|---|---:| +| v0.33.0 (via64) | 91.5 µs | +| `logCR` in Go | 82 µs | +| **`log32CRAVX2`** | **31.1 µs** | +| NumPy 2.5.4, `np.log(a, out=o)` | 31.0 µs | + +Through the API (`BenchmarkDTypeExpLog`, allocating), against v0.33.0, +`abbench` 8 interleaved rounds with load at most 8: −65% at 16 Ki (104.6 → +37.1 µs) and −68% at 1 Mi (6.74 → 2.16 ms), p = 0.000. + +The four gathers per block are not the bottleneck: replacing the `tl` gather +by a register copy saves at most 8%. Doing better than NumPy needs a +different algorithm, not a tweak of this one. + +**Open defect:** float64 `Log` is 1.23× slower than NumPy at 16 Ki on Zen 3 +(85 µs against 69 µs), while float64 `Exp` is 16% faster. The float32 kernels +for the other architectures are still to write; there `Exp` and `Log` run +the Go transcriptions. ## SIMD coverage diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index f1bd76e..4bf0918 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -56,6 +56,7 @@ func main() { f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) f.Add(amd64.FeatureProbe("hasAVX2", amd64.AVX2)) f.Add(exp32CRKernel()) + f.Add(log32CRKernel()) f.Add(blocks8Kernel()) f.Add(bin32Kernel("add32SSE2", "ADDPS", "ADDSS")) f.Add(bin32Kernel("sub32SSE2", "SUBPS", "SUBSS")) @@ -446,6 +447,108 @@ func exp32CRKernel() *emit.Function { return b.Func() } +// log32CRKernel builds log32CRAVX2(dst, src *float32, n int, k +// *logCRConsts) int: logCR's fast path (logcr.go, CORE-MATH's logf) on eight +// float32 at a time, n a positive multiple of 8. Per block: +// +// - the integer part in 32-bit lanes: m = ux & 0x7fffff, j = (m+2^16)>>17 +// (at most 64 for any ux, so the gathers stay in the 65-entry tables), +// e = (ux>>23) - 127; +// - per four lanes, in float64 with logCR's unfused operations in its order +// (VMULPD then VADDPD, never FMA): tz from (m | 1023<<23)<<29, tr[j] and +// tl[j] gathered (VGATHERDPD), z = tz*tr - 1, r, and the bounds r and +// r + 0x1.f06p-33 rounded to float32; +// - x == 1 is 0 (logCR's early return). +// +// A block with an input that is not a positive normal float32, or a lane +// whose bounds round apart, is not stored: the kernel returns the count of +// floats it wrote, and the Go wrapper finishes that block with logCR. Needs +// AVX2 (gathers, 256-bit integer ops). +func log32CRKernel() *emit.Function { + sig := amd64.Layout( + []string{"dst", "src", "n", "k"}, + []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64, amd64.Ptr}, + []string{"done"}, []amd64.Type{amd64.Int64}, + ) + const ( + one, ln2, b0, b1, b2, w = 0, 32, 64, 96, 128, 160 + sub, maxv, mmask, jadd, bias, oneBits = 192, 224, 256, 288, 320, 352 + expOr = 384 + tr, tl = 416, 936 + ) + b := amd64.NewFunc("log32CRAVX2", sig, 0) + b.LoadArg("dst", "DI").LoadArg("src", "SI").LoadArg("n", "CX").LoadArg("k", "R8") + b.Raw(fmt.Sprintf("LEAQ %d(R8), R9", tr)) + b.Raw(fmt.Sprintf("LEAQ %d(R8), R10", tl)) + b.Raw("XORQ AX, AX") + // half runs the float64 part on the four lanes whose j is in X2, mantissa + // bits (zero-extended to 64) in Y10 and exponent (as float64) in Y11, + // leaving the bounds in ub and lb. Temporaries Y4-Y9, Y12. + half := func(ub, lb string) { + b.Raw(fmt.Sprintf("VPOR %d(R8), Y10, Y10", expOr)) + b.Raw("VPSLLQ $29, Y10, Y10") // tz, in [1,2) + b.Raw("VPCMPEQQ Y12, Y12, Y12") // gather every lane + b.Raw("VGATHERDPD Y12, (R9)(X2*8), Y4") // tr[j] + b.Raw("VPCMPEQQ Y12, Y12, Y12") + b.Raw("VGATHERDPD Y12, (R10)(X2*8), Y5") // tl[j] + b.Raw("VMULPD Y4, Y10, Y4") // tz*tr + b.Raw(fmt.Sprintf("VSUBPD %d(R8), Y4, Y4", one)) // z + b.Raw("VMULPD Y4, Y4, Y6") // z2 + b.Raw(fmt.Sprintf("VMULPD %d(R8), Y11, Y7", ln2)) // e*ln2 + b.Raw("VADDPD Y5, Y7, Y7") // + tl + b.Raw(fmt.Sprintf("VMULPD %d(R8), Y4, Y8", b0)) + b.Raw("VADDPD Y8, Y7, Y7") // + z*b0 + b.Raw(fmt.Sprintf("VMULPD %d(R8), Y4, Y8", b2)) + b.Raw(fmt.Sprintf("VADDPD %d(R8), Y8, Y8", b1)) // b1 + z*b2 + b.Raw("VMULPD Y8, Y6, Y8") + b.Raw("VADDPD Y8, Y7, Y7") // r + b.Raw(fmt.Sprintf("VADDPD %d(R8), Y7, Y9", w)) // r + 0x1.f06p-33 + b.Raw(fmt.Sprintf("VCVTPD2PSY Y7, %s", ub)) + b.Raw(fmt.Sprintf("VCVTPD2PSY Y9, %s", lb)) + } + b.Raw("lloop:") + b.Raw("VMOVDQU (SI), Y0") + // Not a positive normal: v = ux - 2^23 outside [0, 0x7effffff]. + b.Raw(fmt.Sprintf("VPSUBD %d(R8), Y0, Y4", sub)) + b.Raw(fmt.Sprintf("VPCMPGTD %d(R8), Y4, Y5", maxv)) // v > 0x7effffff + b.Raw("VPXOR Y6, Y6, Y6") + b.Raw("VPCMPGTD Y4, Y6, Y6") // 0 > v + b.Raw("VPOR Y5, Y6, Y6") + b.Raw("VPTEST Y6, Y6") + b.Raw("JNZ lbail") + b.Raw(fmt.Sprintf("VPCMPEQD %d(R8), Y0, Y13", oneBits)) // x == 1 + b.Raw(fmt.Sprintf("VPAND %d(R8), Y0, Y1", mmask)) // m + b.Raw(fmt.Sprintf("VPADDD %d(R8), Y1, Y2", jadd)) + b.Raw("VPSRLD $17, Y2, Y2") // j + b.Raw("VPSRLD $23, Y0, Y3") + b.Raw(fmt.Sprintf("VPSUBD %d(R8), Y3, Y3", bias)) // e + b.Raw("VPMOVZXDQ X1, Y10") + b.Raw("VCVTDQ2PD X3, Y11") + half("X14", "X15") + b.Raw("VEXTRACTI128 $1, Y1, X1") + b.Raw("VPMOVZXDQ X1, Y10") + b.Raw("VEXTRACTI128 $1, Y3, X3") + b.Raw("VCVTDQ2PD X3, Y11") + b.Raw("VEXTRACTI128 $1, Y2, X2") + half("X1", "X3") + b.Raw("VINSERTI128 $1, X1, Y14, Y14") // ub, eight lanes + b.Raw("VINSERTI128 $1, X3, Y15, Y15") // lb + b.Raw("VPCMPEQD Y15, Y14, Y15") // the bounds agree + b.Raw("VPOR Y13, Y15, Y15") // or x == 1 + b.Raw("VMOVMSKPS Y15, DX") + b.Raw("CMPL DX, $0xff") + b.Raw("JNE lbail") + b.Raw("VPANDN Y14, Y13, Y14") // x == 1 -> +0 + b.Raw("VMOVDQU Y14, (DI)") + b.Raw("ADDQ $32, SI").Raw("ADDQ $32, DI").Raw("ADDQ $8, AX") + b.Raw("CMPQ AX, CX").Raw("JLT lloop") + b.Raw("lbail:") + b.Raw("VZEROUPPER") + b.StoreRet("AX", "done") + b.Ret() + return b.Func() +} + // bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i], // 16 floats per iteration over four XMM registers with the packed op, then a // scalar tail with sca. Each lane is one IEEE float32 operation, so the result diff --git a/internal/kernels/expcr_other.go b/internal/kernels/expcr_other.go index ed70ad1..80b6267 100644 --- a/internal/kernels/expcr_other.go +++ b/internal/kernels/expcr_other.go @@ -5,3 +5,6 @@ package kernels // exp32Fast is the correctly rounded float32 exp: expCR in Go on these // architectures (no SIMD kernel yet). func exp32Fast(dst, src []float32) { exp32CR(dst, src) } + +// log32Fast is the correctly rounded float32 log: logCR in Go here. +func log32Fast(dst, src []float32) { logCRSlice(dst, src) } diff --git a/internal/kernels/logcr.go b/internal/kernels/logcr.go new file mode 100644 index 0000000..01e00f8 --- /dev/null +++ b/internal/kernels/logcr.go @@ -0,0 +1,138 @@ +package kernels + +import ( + "math" + "math/bits" +) + +// logCR is CORE-MATH's correctly rounded binary32 logarithm (Alexei Sibidanov +// and Paul Zimmermann, src/binary32/log/logf.c, MIT), one C operation per Go +// operation. x = 2^e·m, m in [1,2); tr[j] ≈ 1/m on 64 intervals so that +// z = m·tr[j] - 1 is small; log x = e·ln2 + tl[j] + log1p(z), with +// tl[j] = -log(tr[j]). A degree-3 polynomial gives a lower bound r of the +// exact value with r + 0x1.f06p-33 an upper bound; when both round to the +// same float32 that float32 is the answer. Otherwise (762 750 of the 2^31 +// positive finite inputs, about 1 in 2800, measured by TestLogCRMatchesLog32) +// a degree-8 polynomial with a split ln2 decides it. +// +// Every product is converted explicitly to float64, which Go defines as +// rounding it, so no architecture fuses it into an FMA: the operations are +// the C source's, unfused, everywhere. TestLogCRMatchesLog32 holds the result +// to log32, already proven correctly rounded, over every float32. +func logCR(x float32) float32 { + ux := math.Float32bits(x) + if ux < 1<<23 || ux >= 0x7f800000 { + if ux == 0 || ux >= 0x7f800000 { + return logCRSpecial(x) + } + n := bits.LeadingZeros32(ux) - 8 // subnormal: normalise the mantissa + ux <<= n + ux -= uint32(n) << 23 + } + if ux == 127<<23 { + return 0 + } + m := ux & (1<<23 - 1) + j := (m + 1<<(23-7)) >> (23 - 6) + e := float64(int32(ux)>>23 - 127) + z := float64(math.Float64frombits((uint64(m)|1023<<23)<<(52-23))*logCRTr[j]) - 1 + z2 := float64(z * z) + r := ((float64(e*0x1.62e42fefa39efp-1) + logCRTl[j]) + float64(z*logCRB[0])) + + float64(z2*(logCRB[1]+float64(z*logCRB[2]))) + ub, lb := float32(r), float32(r+0x1.f06p-33) + if ub == lb { + return ub + } + return logCRSlow(x, z, z2, e, j) +} + +// logCRSlow is logf.c's second step, for the inputs whose bounds straddle a +// float32 rounding boundary. +func logCRSlow(x float32, z, z2, e float64, j uint32) float32 { + c := &logCRC + f := float64(z2 * ((c[0] + float64(z*c[1])) + float64(z2*((c[2]+float64(z*c[3]))+ + float64(z2*(c[4]+float64(z*c[5])+float64(z2*c[6]))))))) + if math.Abs(float64(x-1)) < 0x1p-10 { + return float32(z + f) + } + f -= float64(0x1.0ca86c3898dp-49 * e) + f += z + f += logCRTl[j] - logCRTl[0] + el := float64(e * 0x1.62e42fefa3ap-1) + r := el + f + ub := float32(r) + if math.Float64bits(r)&(1<<28-1) == 0 { + dr := (el - r) + f + r += float64(dr * 64) + ub = float32(r) + } + return ub +} + +// logCRSpecial is logf.c's as_special: ±0 → -Inf, +Inf → +Inf, NaN → NaN, +// negatives → NaN. +func logCRSpecial(x float32) float32 { + switch ux := math.Float32bits(x); { + case ux<<1 == 0: + return float32(math.Inf(-1)) + case ux == 0x7f800000: + return x + case ux<<1 > 0xff000000: + return x + x + } + return float32(math.NaN()) +} + +// logCRSlice writes logCR(src[i]) into dst[i]. +func logCRSlice(dst, src []float32) { + src = src[:len(dst)] + for i, x := range src { + dst[i] = logCR(x) + } +} + +var logCRTr = [65]float64{ + 0x1p+0, 0x1.f81f82p-1, 0x1.f07c1fp-1, 0x1.e9131acp-1, + 0x1.e1e1e1ep-1, 0x1.dae6077p-1, 0x1.d41d41dp-1, 0x1.cd85689p-1, + 0x1.c71c71cp-1, 0x1.c0e0704p-1, 0x1.bacf915p-1, 0x1.b4e81b5p-1, + 0x1.af286bdp-1, 0x1.a98ef6p-1, 0x1.a41a41ap-1, 0x1.9ec8e95p-1, + 0x1.999999ap-1, 0x1.948b0fdp-1, 0x1.8f9c19p-1, 0x1.8acb90fp-1, + 0x1.8618618p-1, 0x1.8181818p-1, 0x1.7d05f41p-1, 0x1.78a4c81p-1, + 0x1.745d174p-1, 0x1.702e05cp-1, 0x1.6c16c17p-1, 0x1.6816817p-1, + 0x1.642c859p-1, 0x1.605816p-1, 0x1.5c9882cp-1, 0x1.58ed231p-1, + 0x1.5555555p-1, 0x1.51d07ebp-1, 0x1.4e5e0a7p-1, 0x1.4afd6ap-1, + 0x1.47ae148p-1, 0x1.446f865p-1, 0x1.4141414p-1, 0x1.3e22cbdp-1, + 0x1.3b13b14p-1, 0x1.3813814p-1, 0x1.3521cfbp-1, 0x1.323e34ap-1, + 0x1.2f684bep-1, 0x1.2c9fb4ep-1, 0x1.29e412ap-1, 0x1.27350b9p-1, + 0x1.2492492p-1, 0x1.21fb781p-1, 0x1.1f7047ep-1, 0x1.1cf06aep-1, + 0x1.1a7b961p-1, 0x1.1811812p-1, 0x1.15b1e5fp-1, 0x1.135c811p-1, + 0x1.1111111p-1, 0x1.0ecf56cp-1, 0x1.0c9715p-1, 0x1.0a6810ap-1, + 0x1.0842108p-1, 0x1.0624dd3p-1, 0x1.041041p-1, 0x1.0204081p-1, 0.5, +} + +var logCRTl = [65]float64{ + -0x1.3b40815cd0628p-45, 0x1.fc0a890fbb514p-7, 0x1.f829b1e780b98p-6, 0x1.77458f532c948p-5, + 0x1.f0a30c2114ef2p-5, 0x1.341d793bbc7f7p-4, 0x1.6f0d28d256172p-4, 0x1.a926d3a6acb89p-4, + 0x1.e2707722ae90cp-4, 0x1.0d77e7a90896cp-3, 0x1.29552f6fff036p-3, 0x1.44d2b6c5b7831p-3, + 0x1.5ff306ee78ee7p-3, 0x1.7ab890410d41cp-3, 0x1.9525a9e3451c7p-3, 0x1.af3c94ed0bb06p-3, + 0x1.c8ff7c59a9535p-3, 0x1.e27076d5aedf9p-3, 0x1.fb9186b5e393ep-3, 0x1.0a324e38b8e6dp-2, + 0x1.1675cacaba398p-2, 0x1.22941fc0f76efp-2, 0x1.2e8e2bc311abap-2, 0x1.3a64c56b14373p-2, + 0x1.4618bc31c5c4cp-2, 0x1.51aad874df5b7p-2, 0x1.5d1bdbea80754p-2, 0x1.686c81d331238p-2, + 0x1.739d7f6dbcd9p-2, 0x1.7eaf83c82ad4dp-2, 0x1.89a3385813fe4p-2, 0x1.947941aa91484p-2, + 0x1.9f323edbf95d5p-2, 0x1.a9cec9a4205d3p-2, 0x1.b44f77c5c8cecp-2, 0x1.beb4d9ea71905p-2, + 0x1.c8ff7c69a97abp-2, 0x1.d32fe7f38e95fp-2, 0x1.dd46a0501c22ap-2, 0x1.e7442617e8511p-2, + 0x1.f128f5eaf0476p-2, 0x1.faf588dd8f0a8p-2, 0x1.02552a5edcfc4p-1, 0x1.0723e5c64de05p-1, + 0x1.0be72e3852947p-1, 0x1.109f39d554b5cp-1, 0x1.154c3d2c4d4aep-1, 0x1.19ee6b38bc834p-1, + 0x1.1e85f5ef03f95p-1, 0x1.23130d7fabe07p-1, 0x1.2795e1219afep-1, 0x1.2c0e9ec9c8d5p-1, + 0x1.307d7337f0f83p-1, 0x1.34e289cb4e098p-1, 0x1.393e0d42e28dep-1, 0x1.3d9026ad555bfp-1, + 0x1.41d8fe8667173p-1, 0x1.4618bc1ec5d87p-1, 0x1.4a4f85d303d8p-1, 0x1.4e7d8127f5a75p-1, + 0x1.52a2d26dbc47p-1, 0x1.56bf9d597f25ep-1, 0x1.5ad404cb59df2p-1, 0x1.5ee02a928153ap-1, + 0x1.62e42fefa38b4p-1, +} + +var logCRB = [3]float64{0x1.00000006342eap+0, -0x1.0001f7fdc3977p-1, 0x1.554a4e5cae9cfp-2} + +var logCRC = [7]float64{ + -0x1p-1, 0x1.55555555571cap-2, -0x1.0000000002d85p-2, 0x1.9999987d0c963p-3, + -0x1.555554059a8bbp-3, 0x1.24aebcf71a38fp-3, -0x1.001c73915d758p-3, +} diff --git a/internal/kernels/logcr_amd64.go b/internal/kernels/logcr_amd64.go new file mode 100644 index 0000000..e0d005b --- /dev/null +++ b/internal/kernels/logcr_amd64.go @@ -0,0 +1,52 @@ +package kernels + +// logCRConsts is the constant block log32CRAVX2 reads (asmgen/amd64/gen.go, +// log32CRKernel), at the offsets the kernel uses. +type logCRConsts struct { + one, ln2, b0, b1, b2, w [4]float64 // offsets 0, 32, 64, 96, 128, 160 + sub, maxv, mmask, jadd, bias, oneBits [8]uint32 // 192, 224, 256, 288, 320, 352 + expOr [4]uint64 // 384 + tr, tl [65]float64 // 416, 936 +} + +var logCRK = func() *logCRConsts { + k := new(logCRConsts) + for i := range 4 { + k.one[i], k.ln2[i], k.w[i] = 1, 0x1.62e42fefa39efp-1, 0x1.f06p-33 + k.b0[i], k.b1[i], k.b2[i] = logCRB[0], logCRB[1], logCRB[2] + k.expOr[i] = 1023 << 23 + } + for i := range 8 { + k.sub[i], k.maxv[i], k.mmask[i] = 1<<23, 0x7effffff, 1<<23-1 + k.jadd[i], k.bias[i], k.oneBits[i] = 1<<16, 127, 0x3f800000 + } + k.tr, k.tl = logCRTr, logCRTl + return k +}() + +// useLog32AVX2 selects the AVX2 kernel; a var so tests can force the Go path. +var useLog32AVX2 = hasAVX2() + +// log32Fast is the correctly rounded float32 log: blocks of 8 on the AVX2 +// kernel, which hands back each block it cannot finish (an input that is not +// a positive normal float32, or bounds that round apart, about 1 lane in +// 2800) to logCR, as is the tail. Both run logCR's operations in its order, +// so they agree bit for bit. +func log32Fast(dst, src []float32) { + src = src[:len(dst)] + i := 0 + if useLog32AVX2 { + n := len(dst) &^ 7 + for i < n { + i += log32CRAVX2(&dst[i], &src[i], n-i, logCRK) + if i < n { + logCRSlice(dst[i:i+8], src[i:i+8]) + i += 8 + } + } + } + logCRSlice(dst[i:], src[i:]) +} + +//go:noescape +func log32CRAVX2(dst, src *float32, n int, k *logCRConsts) (done int) diff --git a/internal/kernels/logcr_amd64_test.go b/internal/kernels/logcr_amd64_test.go new file mode 100644 index 0000000..624f445 --- /dev/null +++ b/internal/kernels/logcr_amd64_test.go @@ -0,0 +1,91 @@ +package kernels + +import ( + "math" + "math/rand" + "testing" + "unsafe" +) + +// TestLogCRConstsLayout holds logCRConsts to the offsets log32CRKernel uses. +func TestLogCRConstsLayout(t *testing.T) { + var k logCRConsts + for _, c := range []struct { + name string + got, want uintptr + }{ + {"one", unsafe.Offsetof(k.one), 0}, {"ln2", unsafe.Offsetof(k.ln2), 32}, + {"b0", unsafe.Offsetof(k.b0), 64}, {"b1", unsafe.Offsetof(k.b1), 96}, + {"b2", unsafe.Offsetof(k.b2), 128}, {"w", unsafe.Offsetof(k.w), 160}, + {"sub", unsafe.Offsetof(k.sub), 192}, {"maxv", unsafe.Offsetof(k.maxv), 224}, + {"mmask", unsafe.Offsetof(k.mmask), 256}, {"jadd", unsafe.Offsetof(k.jadd), 288}, + {"bias", unsafe.Offsetof(k.bias), 320}, {"oneBits", unsafe.Offsetof(k.oneBits), 352}, + {"expOr", unsafe.Offsetof(k.expOr), 384}, {"tr", unsafe.Offsetof(k.tr), 416}, + {"tl", unsafe.Offsetof(k.tl), 936}, + } { + if c.got != c.want { + t.Errorf("%s at %d, the kernel reads %d", c.name, c.got, c.want) + } + } +} + +// TestLog32AVX2MatchesGo holds log32Fast (AVX2 blocks, Go for the blocks it +// hands back and the tail) to logCR bit for bit (NaN to NaN): random bit +// patterns (a third of them negative, NaN or subnormal, so most blocks bail), +// random positive normals (few bail), 1.0 runs, the special values, and every +// length 0..70. +func TestLog32AVX2MatchesGo(t *testing.T) { + if !useLog32AVX2 { + t.Skip("no AVX2 here") + } + r := rand.New(rand.NewSource(5)) + src := make([]float32, 0, 300000) + for _, v := range []float32{1, 1, 1, 1, 1, 1, 1, 1, 0, float32(math.Copysign(0, -1)), -1, + float32(math.Inf(1)), float32(math.Inf(-1)), float32(math.NaN()), math.SmallestNonzeroFloat32, + 0x1p-126, math.MaxFloat32, 0.99999994, 1.0000001, 2, 0.5} { + src = append(src, v) + } + for len(src) < 100000 { + src = append(src, math.Float32frombits(r.Uint32())) + } + for len(src) < cap(src) { + src = append(src, math.Float32frombits(0x00800000+uint32(r.Int63n(0x7f000000)))) + } + want := make([]float32, len(src)) + logCRSlice(want, src) + got := make([]float32, len(src)) + log32Fast(got, src) + same := func(a, b float32) bool { return math.Float32bits(a) == math.Float32bits(b) || (a != a && b != b) } + for i := range src { + if !same(got[i], want[i]) { + t.Fatalf("log(0x%08x): AVX2 0x%08x, Go 0x%08x", math.Float32bits(src[i]), math.Float32bits(got[i]), math.Float32bits(want[i])) + } + } + for n := 0; n <= 70; n++ { + d := make([]float32, n) + log32Fast(d, src[:n]) + for i := range d { + if !same(d[i], want[i]) { + t.Fatalf("length %d [%d]: 0x%08x, want 0x%08x", n, i, math.Float32bits(d[i]), math.Float32bits(want[i])) + } + } + } +} + +// TestLog32AVX2InPlace runs the kernel with dst == src: a block it hands +// back must still hold its inputs. +func TestLog32AVX2InPlace(t *testing.T) { + r := rand.New(rand.NewSource(6)) + src := make([]float32, 4099) + for i := range src { + src[i] = math.Float32frombits(r.Uint32() &^ (1 << 31)) // NaN, Inf and subnormals among them + } + want := make([]float32, len(src)) + logCRSlice(want, src) + log32Fast(src, src) + for i := range src { + if math.Float32bits(src[i]) != math.Float32bits(want[i]) && !(src[i] != src[i] && want[i] != want[i]) { + t.Fatalf("[%d]: 0x%08x, want 0x%08x", i, math.Float32bits(src[i]), math.Float32bits(want[i])) + } + } +} diff --git a/internal/kernels/logcr_fence_amd64_test.go b/internal/kernels/logcr_fence_amd64_test.go new file mode 100644 index 0000000..26c1604 --- /dev/null +++ b/internal/kernels/logcr_fence_amd64_test.go @@ -0,0 +1,20 @@ +//go:build linux || darwin + +package kernels + +import "testing" + +// TestLog32AVX2StaysInside fences src and dst of the AVX2 log kernel +// (fences32 needs mmap/mprotect, hence the build tag of fence_test.go). +func TestLog32AVX2StaysInside(t *testing.T) { + if !useLog32AVX2 { + t.Skip("no AVX2 here") + } + for n := 8; n <= 64; n += 8 { + for _, s := range fences32(t, n) { + for _, d := range fences32(t, n) { + log32CRAVX2(&d[0], &s[0], n, logCRK) + } + } + } +} diff --git a/internal/kernels/logcr_test.go b/internal/kernels/logcr_test.go new file mode 100644 index 0000000..489571f --- /dev/null +++ b/internal/kernels/logcr_test.go @@ -0,0 +1,127 @@ +package kernels + +import ( + "math" + "math/bits" + "runtime" + "sync" + "sync/atomic" + "testing" +) + +// TestLogCRMatchesLog32 holds logCR to log32 bit for bit (NaN to NaN) on +// every float32 input (NDARRAY_EXHAUSTIVE=1; otherwise every 65537th). +// log32 is proven correctly rounded over all 2^32 inputs by +// TestLog32CorrectlyRounded, so equality everywhere proves logCR is too. It +// also counts the inputs logCRSlow decides. +func TestLogCRMatchesLog32(t *testing.T) { + stride := exhaustiveStride() + const batch = 4096 + var next atomic.Uint64 + var bad, slow, checked atomic.Int64 + var mu sync.Mutex + var first []string + var wg sync.WaitGroup + for range runtime.GOMAXPROCS(0) { + wg.Add(1) + go func() { + defer wg.Done() + xs := make([]float32, 0, batch) + want := make([]float32, batch) + for { + start := next.Add(batch*stride) - batch*stride + if start >= 1<<32 { + return + } + xs = xs[:0] + for b := start; b < start+batch*stride && b < 1<<32; b += stride { + xs = append(xs, math.Float32frombits(uint32(b))) + } + log32(want[:len(xs)], xs) + for i, x := range xs { + checked.Add(1) + got := logCR(x) + if logCRTakesSlow(x) { + slow.Add(1) + } + w := want[i] + if math.Float32bits(got) != math.Float32bits(w) && !(got != got && w != w) { + bad.Add(1) + mu.Lock() + if len(first) < 8 { + first = append(first, "logCR("+formatF32(x)+") = "+formatF32(got)+", log32 "+formatF32(w)) + } + mu.Unlock() + } + } + } + }() + } + wg.Wait() + t.Logf("%d inputs checked, %d through the slow path, %d differ from log32", checked.Load(), slow.Load(), bad.Load()) + for _, s := range first { + t.Error(s) + } +} + +// logCRTakesSlow repeats logCR's fast path and reports whether its bounds +// straddle a rounding boundary. +func logCRTakesSlow(x float32) bool { + ux := math.Float32bits(x) + if ux == 0 || ux >= 0x7f800000 || ux == 127<<23 { + return false + } + if ux < 1<<23 { + n := bits.LeadingZeros32(ux) - 8 + ux <<= n + ux -= uint32(n) << 23 + } + m := ux & (1<<23 - 1) + j := (m + 1<<(23-7)) >> (23 - 6) + e := float64(int32(ux)>>23 - 127) + z := float64(math.Float64frombits((uint64(m)|1023<<23)<<(52-23))*logCRTr[j]) - 1 + z2 := float64(z * z) + r := ((float64(e*0x1.62e42fefa39efp-1) + logCRTl[j]) + float64(z*logCRB[0])) + + float64(z2*(logCRB[1]+float64(z*logCRB[2]))) + return float32(r) != float32(r+0x1.f06p-33) +} + +// BenchmarkLog32Kernels compares the float32 log kernels on 16 Ki inputs in +// [0.25, 24.25), the range of benchmarks/BenchmarkDTypeExpLog. +func BenchmarkLog32Kernels(b *testing.B) { + src := make([]float32, 16<<10) + for i := range src { + src[i] = 0.25 + 24*float32(i)/float32(len(src)) + } + dst := make([]float32, len(src)) + for _, k := range []struct { + name string + f func(dst, src []float32) + }{{"via64", log32}, {"logCR", logCRSlice}, {"fast", log32Fast}} { + b.Run(k.name, func(b *testing.B) { + b.SetBytes(int64(4 * len(src))) + for b.Loop() { + k.f(dst, src) + } + }) + } +} + +func TestLogCRSpecial(t *testing.T) { + for _, c := range []struct{ x, want float32 }{ + {0, float32(math.Inf(-1))}, + {float32(math.Copysign(0, -1)), float32(math.Inf(-1))}, + {float32(math.Inf(1)), float32(math.Inf(1))}, + {1, 0}, + {math.SmallestNonzeroFloat32, -103.27893}, + } { + if got := logCR(c.x); got != c.want { + t.Errorf("logCR(%v) = %v, want %v", c.x, got, c.want) + } + } + for _, x := range []float32{-1, float32(math.Inf(-1)), float32(math.NaN()), -math.SmallestNonzeroFloat32} { + if got := logCR(x); got == got { + t.Errorf("logCR(%v) = %v, want NaN", x, got) + } + } +} diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index de6f8f0..0a86822 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -1781,6 +1781,95 @@ TEXT ·exp32CRAVX2(SB), NOSPLIT, $0-32 VZEROUPPER RET +TEXT ·log32CRAVX2(SB), NOSPLIT, $0-40 + MOVQ dst+0(FP), DI + MOVQ src+8(FP), SI + MOVQ n+16(FP), CX + MOVQ k+24(FP), R8 + LEAQ 416(R8), R9 + LEAQ 936(R8), R10 + XORQ AX, AX + lloop: + VMOVDQU (SI), Y0 + VPSUBD 192(R8), Y0, Y4 + VPCMPGTD 224(R8), Y4, Y5 + VPXOR Y6, Y6, Y6 + VPCMPGTD Y4, Y6, Y6 + VPOR Y5, Y6, Y6 + VPTEST Y6, Y6 + JNZ lbail + VPCMPEQD 352(R8), Y0, Y13 + VPAND 256(R8), Y0, Y1 + VPADDD 288(R8), Y1, Y2 + VPSRLD $17, Y2, Y2 + VPSRLD $23, Y0, Y3 + VPSUBD 320(R8), Y3, Y3 + VPMOVZXDQ X1, Y10 + VCVTDQ2PD X3, Y11 + VPOR 384(R8), Y10, Y10 + VPSLLQ $29, Y10, Y10 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R9)(X2*8), Y4 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R10)(X2*8), Y5 + VMULPD Y4, Y10, Y4 + VSUBPD 0(R8), Y4, Y4 + VMULPD Y4, Y4, Y6 + VMULPD 32(R8), Y11, Y7 + VADDPD Y5, Y7, Y7 + VMULPD 64(R8), Y4, Y8 + VADDPD Y8, Y7, Y7 + VMULPD 128(R8), Y4, Y8 + VADDPD 96(R8), Y8, Y8 + VMULPD Y8, Y6, Y8 + VADDPD Y8, Y7, Y7 + VADDPD 160(R8), Y7, Y9 + VCVTPD2PSY Y7, X14 + VCVTPD2PSY Y9, X15 + VEXTRACTI128 $1, Y1, X1 + VPMOVZXDQ X1, Y10 + VEXTRACTI128 $1, Y3, X3 + VCVTDQ2PD X3, Y11 + VEXTRACTI128 $1, Y2, X2 + VPOR 384(R8), Y10, Y10 + VPSLLQ $29, Y10, Y10 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R9)(X2*8), Y4 + VPCMPEQQ Y12, Y12, Y12 + VGATHERDPD Y12, (R10)(X2*8), Y5 + VMULPD Y4, Y10, Y4 + VSUBPD 0(R8), Y4, Y4 + VMULPD Y4, Y4, Y6 + VMULPD 32(R8), Y11, Y7 + VADDPD Y5, Y7, Y7 + VMULPD 64(R8), Y4, Y8 + VADDPD Y8, Y7, Y7 + VMULPD 128(R8), Y4, Y8 + VADDPD 96(R8), Y8, Y8 + VMULPD Y8, Y6, Y8 + VADDPD Y8, Y7, Y7 + VADDPD 160(R8), Y7, Y9 + VCVTPD2PSY Y7, X1 + VCVTPD2PSY Y9, X3 + VINSERTI128 $1, X1, Y14, Y14 + VINSERTI128 $1, X3, Y15, Y15 + VPCMPEQD Y15, Y14, Y15 + VPOR Y13, Y15, Y15 + VMOVMSKPS Y15, DX + CMPL DX, $0xff + JNE lbail + VPANDN Y14, Y13, Y14 + VMOVDQU Y14, (DI) + ADDQ $32, SI + ADDQ $32, DI + ADDQ $8, AX + CMPQ AX, CX + JLT lloop + lbail: + VZEROUPPER + MOVQ AX, done+32(FP) + RET + TEXT ·blocks8SSE2(SB), NOSPLIT, $0-24 MOVQ r+0(FP), DI MOVQ a+8(FP), SI diff --git a/internal/kernels/via64.go b/internal/kernels/via64.go index 3ef320f..615ca01 100644 --- a/internal/kernels/via64.go +++ b/internal/kernels/via64.go @@ -1,98 +1,31 @@ package kernels -import "math" - // Correctly rounded float32 Exp and Log. // // float32 Exp and Log went through math.Exp and math.Log per element, 6-14x // slower than NumPy's float32 SIMD on one Zen 3 core. Both now compute in // float64 and round once, and both are proven correctly rounded over every -// float32 input (via64_exhaustive_test.go, expcr_test.go): +// float32 input (via64_exhaustive_test.go, expcr_test.go, logcr_test.go): // -// - Exp runs expCR (expcr.go), Arm's correctly rounded expf algorithm, on -// an AVX2 kernel where there is one (expcr_amd64.go), in Go elsewhere; -// - Log widens to this package's float64 Log (0.504 ULP) through a -// 256-element stack buffer, rounds once, and takes the 24 inputs whose -// float64 result lies next to a float32 rounding midpoint from a table. +// - Exp runs expCR (expcr.go), Arm's correctly rounded expf algorithm; +// - Log runs logCR (logcr.go), CORE-MATH's correctly rounded logf; // -// Nothing is allocated. Log is called directly, not through a func value, -// which would make the buffer escape. +// each on an AVX2 kernel where there is one (expcr_amd64.go, logcr_amd64.go), +// in Go elsewhere. Nothing is allocated. const via64Chunk = 256 -// log32 writes the correctly rounded log(src[i]) into dst[i]. -func log32(dst, src []float32) { - var buf [via64Chunk]float64 - src = src[:len(dst)] - for i := 0; i < len(dst); i += via64Chunk { - b := buf[:min(via64Chunk, len(dst)-i)] - s := src[i : i+len(b)] - for j := range b { - b[j] = float64(s[j]) - } - Log(b, b) - d := dst[i : i+len(b)] - for j, v := range b { - f := float32(v) - // Within 8 float64 ULP of a float32 rounding midpoint (the 29 - // bits float32 drops are 0x10000000 +- 8): the double rounding - // may be wrong, so take the table's answer. - if math.Float64bits(v)&(1<<29-1)-(1<<28-8) < 16 { - f = log32Exception(s[j], f) - } - d[j] = f - } - } -} - -// log32Exceptions are the float32 inputs whose float64 log lies within 2 -// float64 ULP of a float32 rounding midpoint, sorted, each with its correctly -// rounded log decided by a 300-bit reference. The exhaustive test lists them -// (NDARRAY_EXHAUSTIVE=1 NDARRAY_DUMP_HARD=1, TestLog32CorrectlyRounded): 24 -// inputs, 5 of which rounded to the wrong float32 without this table. Log's -// float64 arithmetic is the same on every architecture (its multiply-add is -// math.FMA), so the list is too. -var log32Exceptions = [...][2]uint32{ - {0x0dc8bba4, 0xc289bac4}, {0x111c87f8, 0xc28085df}, {0x1a8446cb, 0xc24d0a45}, - {0x1f116ab8, 0xc233b53e}, {0x28e3fa26, 0xc1fa75e0}, {0x29e6126b, 0xc1ef4c02}, - {0x29fd22f8, 0xc1ee8859}, {0x2c4c24b7, 0xc1d48710}, {0x2e492984, 0xc1be76ee}, - {0x38dcbe38, 0xc1128ba4}, {0x3bf86ef0, 0xc09c399e}, {0x3c413d3a, 0xc08e158f}, - {0x41178feb, 0x400fe5e7}, {0x464d5b2b, 0x4117bcf1}, {0x4665a9a6, 0x41198725}, - {0x4c5d65a5, 0x418f034b}, {0x4d604ebe, 0x419a352c}, {0x4e85f412, 0x41a6b811}, - {0x5ee8984e, 0x422e4a21}, {0x65d890d3, 0x4254d1f9}, {0x66a8c860, 0x42595e46}, - {0x66abbd63, 0x4259700e}, {0x6f31a8ec, 0x42845a89}, {0x79e7ec37, 0x42a1ffb7}, -} - -// log32Exception returns the table's correctly rounded log of x if x is in -// log32Exceptions, and f otherwise. -func log32Exception(x, f float32) float32 { - bits := math.Float32bits(x) - lo, hi := 0, len(log32Exceptions) - for lo < hi { - m := (lo + hi) / 2 - switch e := log32Exceptions[m]; { - case e[0] == bits: - return math.Float32frombits(e[1]) - case e[0] < bits: - lo = m + 1 - default: - hi = m - } - } - return f -} - // Exp32 writes the correctly rounded exp(src[i]) into dst[i]. func Exp32(dst, src []float32) { exp32Fast(dst, src) } // Log32 writes the correctly rounded log(src[i]) into dst[i]. -func Log32(dst, src []float32) { log32(dst, src) } +func Log32(dst, src []float32) { log32Fast(dst, src) } // Exp32P is Exp32, split across cores above ParThreshold. func Exp32P(dst, src []float32) { split32(exp32Fast, dst, src) } // Log32P is Log32, split across cores above ParThreshold. -func Log32P(dst, src []float32) { split32(log32, dst, src) } +func Log32P(dst, src []float32) { split32(log32Fast, dst, src) } func split32(f func(dst, src []float32), dst, src []float32) { n := len(dst) diff --git a/internal/kernels/via64_oracle_test.go b/internal/kernels/via64_oracle_test.go new file mode 100644 index 0000000..659a324 --- /dev/null +++ b/internal/kernels/via64_oracle_test.go @@ -0,0 +1,68 @@ +package kernels + +import "math" + +// log32 is the float32 log of v0.32.0 and v0.33.0, kept as an independent +// oracle for logCR (TestLogCRMatchesLog32): it widens to this package's +// float64 Log (0.504 ULP), rounds once, and takes the 24 inputs whose float64 +// result lies next to a float32 rounding midpoint from a table. It shares no +// code or constant with logCR. +func log32(dst, src []float32) { + var buf [via64Chunk]float64 + src = src[:len(dst)] + for i := 0; i < len(dst); i += via64Chunk { + b := buf[:min(via64Chunk, len(dst)-i)] + s := src[i : i+len(b)] + for j := range b { + b[j] = float64(s[j]) + } + Log(b, b) + d := dst[i : i+len(b)] + for j, v := range b { + f := float32(v) + // Within 8 float64 ULP of a float32 rounding midpoint (the 29 + // bits float32 drops are 0x10000000 +- 8): the double rounding + // may be wrong, so take the table's answer. + if math.Float64bits(v)&(1<<29-1)-(1<<28-8) < 16 { + f = log32Exception(s[j], f) + } + d[j] = f + } + } +} + +// log32Exceptions are the float32 inputs whose float64 log lies within 2 +// float64 ULP of a float32 rounding midpoint, sorted, each with its correctly +// rounded log decided by a 300-bit reference (NDARRAY_EXHAUSTIVE=1 +// NDARRAY_DUMP_HARD=1 on the v0.32.0 TestLog32CorrectlyRounded): 24 inputs, 5 +// of which rounded to the wrong float32 without this table. They stay the +// hardest known inputs for any float32 log (TestLog32Exceptions). +var log32Exceptions = [...][2]uint32{ + {0x0dc8bba4, 0xc289bac4}, {0x111c87f8, 0xc28085df}, {0x1a8446cb, 0xc24d0a45}, + {0x1f116ab8, 0xc233b53e}, {0x28e3fa26, 0xc1fa75e0}, {0x29e6126b, 0xc1ef4c02}, + {0x29fd22f8, 0xc1ee8859}, {0x2c4c24b7, 0xc1d48710}, {0x2e492984, 0xc1be76ee}, + {0x38dcbe38, 0xc1128ba4}, {0x3bf86ef0, 0xc09c399e}, {0x3c413d3a, 0xc08e158f}, + {0x41178feb, 0x400fe5e7}, {0x464d5b2b, 0x4117bcf1}, {0x4665a9a6, 0x41198725}, + {0x4c5d65a5, 0x418f034b}, {0x4d604ebe, 0x419a352c}, {0x4e85f412, 0x41a6b811}, + {0x5ee8984e, 0x422e4a21}, {0x65d890d3, 0x4254d1f9}, {0x66a8c860, 0x42595e46}, + {0x66abbd63, 0x4259700e}, {0x6f31a8ec, 0x42845a89}, {0x79e7ec37, 0x42a1ffb7}, +} + +// log32Exception returns the table's correctly rounded log of x if x is in +// log32Exceptions, and f otherwise. +func log32Exception(x, f float32) float32 { + bits := math.Float32bits(x) + lo, hi := 0, len(log32Exceptions) + for lo < hi { + m := (lo + hi) / 2 + switch e := log32Exceptions[m]; { + case e[0] == bits: + return math.Float32frombits(e[1]) + case e[0] < bits: + lo = m + 1 + default: + hi = m + } + } + return f +}