From 9f6eb260f40bfaee82c3aa3d16a1b2662917ea5a Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 14:59:46 +0100 Subject: [PATCH 01/12] Several references per case: Case.Refs, schema 2, per-reference verdicts Case.Refs []Ref (ID, license, Timer, Eval) with Case.Accuracy: rounds rotate every side through every position, one verdict per reference, a headline against the fastest side, accuracy-aware labels, one issue per case when any reference is faster. A repository with no Refs keeps writing schema 1 byte for byte (golden files recorded before the change). Calibrate discards one untimed first call so that a JIT does not choose iters = 1. Adds the design as docs/design/multiref.md. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- case.go | 21 ++ ci.go | 3 + core_test.go | 3 +- docs/design/multiref.md | 489 +++++++++++++++++++++++++++ golden_test.go | 43 +++ issues.go | 28 +- multiref.go | 433 ++++++++++++++++++++++++ multiref_test.go | 544 +++++++++++++++++++++++++++++++ rbtest/rbtest_test.go | 3 +- report.go | 47 ++- report_test.go | 33 +- resume.go | 8 + runner.go | 124 ++++++- table.go | 99 ++++++ testdata/golden/schema1-table.md | 19 ++ testdata/golden/schema1.json | 89 +++++ 16 files changed, 1947 insertions(+), 39 deletions(-) create mode 100644 docs/design/multiref.md create mode 100644 golden_test.go create mode 100644 multiref.go create mode 100644 multiref_test.go create mode 100644 testdata/golden/schema1-table.md create mode 100644 testdata/golden/schema1.json diff --git a/case.go b/case.go index 7c19019..ec7068a 100644 --- a/case.go +++ b/case.go @@ -48,6 +48,16 @@ type Case struct { // Ref times the reference body, inside the reference's process. Nil for a // case only used by the allocation check. Ref Timer + // Refs are additional references (Julia, Boost, Fortran, R, ...), timed in + // the same rounds as Go and Ref. May be empty: a case with no Refs writes + // the schema 1 files it always wrote. + Refs []Ref + // RefEval evaluates the primary reference on the accuracy points; nil gives + // it no accuracy column. Used only with Accuracy. + RefEval EvalFunc + // Accuracy, when set, scores Go, the primary reference and every Ref that + // has an Eval on the same points, once, before the timing rounds. + Accuracy *Accuracy } // Validate checks a registry: unique names, the required fields, a known parity. @@ -66,6 +76,9 @@ func Validate(cases []Case) error { case c.Parity != ParityExact && c.Parity != ParityTolerance && c.Parity != ParityNone: return fmt.Errorf("case %q: unknown parity %q", c.Name, c.Parity) } + if err := validateRefs(c); err != nil { + return err + } seen[c.Name] = true } return nil @@ -75,7 +88,15 @@ var errNoTime = errors.New("a calibration run took no measurable time") // Calibrate returns the iteration count for which one run of t lasts at least // minDur: it doubles from 1, then scales by the last measurement. +// +// One untimed call comes first and its result is discarded. A reference that +// compiles on its first call (Julia) would otherwise last longer than minDur at +// iters = 1, and Calibrate would settle on single-call samples. The call costs +// one iteration and is harmless for any other reference. func Calibrate(t Timer, minDur time.Duration) (int, error) { + if _, err := t(1); err != nil { + return 0, err + } // 1<<40 on 64-bit targets; half of MaxInt where int is 32 bits (a constant // larger than int does not compile there). const maxIters = min(1<<40, math.MaxInt>>1) diff --git a/ci.go b/ci.go index 13426a6..fb26222 100644 --- a/ci.go +++ b/ci.go @@ -287,6 +287,9 @@ func RunCI(cfg CIConfig) error { for _, p := range CheckAllocs(rep, cfg.Cases) { errs = append(errs, errors.New("allocations: "+p)) } + for _, p := range CheckRefs(rep, cfg.Cases) { + errs = append(errs, errors.New("references: "+p)) + } got, err := os.ReadFile(cfg.Table) if err != nil { return err diff --git a/core_test.go b/core_test.go index f6fb99c..ebae431 100644 --- a/core_test.go +++ b/core_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "strings" "testing" @@ -294,7 +295,7 @@ func TestResumeMergesDeferredRows(t *testing.T) { if err != nil { t.Fatal(err) } - if !m.Valid || len(m.Deferred) != 0 || len(m.Rows) != 2 || m.Rows[0].Rounds[0] != aRounds || len(m.Rows[1].Rounds) != 1 { + if !m.Valid || len(m.Deferred) != 0 || len(m.Rows) != 2 || !reflect.DeepEqual(m.Rows[0].Rounds[0], aRounds) || len(m.Rows[1].Rounds) != 1 { t.Fatalf("merged = %+v", m) } if len(Pending(m, cases)) != 0 { diff --git a/docs/design/multiref.md b/docs/design/multiref.md new file mode 100644 index 0000000..0d03aa8 --- /dev/null +++ b/docs/design/multiref.md @@ -0,0 +1,489 @@ +# refbench: several references per case (Julia, R, Fortran, Boost.Math) + +Status: design, 2026-11-13. Step 1 (core, rbw1, Julia worker, compiled-worker +skeleton) is implemented in the PR that adds this file; later steps are not. + +> **Binding override.** The "Decisions" section of +> [go-scikit/refbench#15](https://github.com/go-scikit/refbench/issues/15) +> (maintainer, 2026-11-13) overrides this document wherever they differ. The +> differences known at step 1: +> +> - **A gap against ANY faster reference is a defect**, even when that +> reference is less accurate. Section 3.2 says the headline is the best among +> the references whose accuracy was scored; the decision says the headline is +> against the fastest reference, and a perf issue is drafted whenever one is +> faster (the `tradeoff` label only describes it). Implemented: `Row.Best` is +> the fastest side by median, the primary included; `refbench issues` drafts +> one issue per case when Go is slower than any side, titled after the +> fastest side Go is slower than. +> - **Q1-Q8 are answered by the decisions**: Julia from the pinned pkgx +> binary (Q2), R through `Rscript` only (Q3), `tradeoff` is a perf defect +> (Q4), SciPy stays primary (Q8). +> - **Step 1 leaves out** the per-kind restriction (`-perf.refs`, `case@refID` +> in the deferred list, section 3.1) and the Python `L` request (section 3.4): +> both need a consumer to design against. The first multi-reference run must +> still reproduce the fixtures' SciPy column bit for bit (`FirstDiff`). +> - `Round` is no longer a comparable struct (it carries a map and a slice): +> compare rounds with `reflect.DeepEqual`. + +Subject: the user asked to go past SciPy: "on peut ne pas se limiter à dépasser +scipy; si d'autres font référence (julia/r/fortran…), on peut tenter de faire +aussi bien, voire mieux." + +## 0. Prior art and scope + +- `priorart --repo go-scikit/refbench "multiple references julia R fortran boost"` + (fleettools v0.108.0, 1 commit stale) exited 1 on **refbench#10** (merged). + I read it: it is "partial runs: retries above the gate, per-row validity, + `-perf.resume`". It matched on the word "references" and does not cover + several references per case. Branches: clean. **No prior work on this + subject.** refbench#10's machinery (per-row validity, `Deferred`, `Merge`, + `CheckResume`) is what this design builds on. +- Code read: go-scikit/refbench `origin/main` 8b6268b (`case.go`, `report.go`, + `runner.go`, `worker.go`, `pyworker.py`, README). +- go-numerics has **three code repositories**: `special`, `optimize` and + `integrate` (`gh repo list go-numerics`). It has no fft, linalg or stats + repository, so those are out of scope here. Each of the three already has + a `docs/roadmap.md` that compares *features* with Julia, R and GSL (special: + 2026-11-11 inventory, `docs/roadmap.md:10-19`). This design adds *timing and + accuracy* against the same references. It does not repeat the roadmaps' + feature lists. + +## 1. What refbench does today (the facts this design must keep) + +| fact | where | +|---|---| +| One `Ref Timer` per `Case`, timed inside the reference's process | `case.go` `Case.Ref`, `Timer` | +| A round is one Go sample and one reference sample, alternating order (even: Go first) | `runner.go` `runCase` | +| Calibration doubles from `iters=1` until a run lasts `MinTime` (100 ms) | `case.go` `Calibrate` | +| Verdict is the median ratio plus a quartile-overlap test | README "Verdict"; `stats.go` `Judge` | +| `perf.json` is schema **1**, decoded with `DisallowUnknownFields` | `report.go` `SchemaVersion`, `ReadReport` | +| `Round{load_before, load_after, go_first, go_ns, ref_ns}` | `report.go` | +| `ref_versions` is a flat `map[string]string` | `report.go` `Report.RefVersions` | +| Python worker: JSON lines; data built by a **NumPy expression** (`setup`); the reference is an `eval`'d expression | `worker.go`, `pyworker.py` | +| Single-thread env: OMP/OPENBLAS/MKL/VECLIB = 1 | `worker.go` `SingleThreadEnv` | +| C references are built by `BuildC` with `-O2 -ffp-contract=off` | `cbuild.go`, README | +| CI mode needs no reference and does no timing | `rbtest`, README "CI mode" | + +Three consequences for several references: + +1. **The data recipe cannot be shared.** `special/bench/perf_test.go:57-67` + builds the inputs as a NumPy string (`np.mod(np.arange(n)*0.618…, 1.0)`). + Julia, R and Fortran cannot evaluate it. The arguments must cross + languages as bytes (§3.4). +2. **Calibration includes the JIT.** A Julia kernel compiles on its first + call. If compiling takes longer than `MinTime` (100 ms is common for a + first call through a package), `Calibrate` returns `iters = 1` at once, + and every sample becomes a single call. A worker therefore needs an + untimed warm-up call before calibration (§3.3). That call is harmless for + Python and C as well. +3. **`DisallowUnknownFields` makes any new field a schema break for old + readers** (§3.6). + +## 2. Reference map + +The license column decides what we may do. **Source** means we may read the +code and port from it. **Black box** means we may run it, time it, and read +its documentation and papers, but never its code. Every license below was +opened, except where a row says "verify". + +### 2.1 Runtimes and how to get them (pkgx, nothing installed yet) + +Recipes were checked in pkgxdev/pantry and in go-zpkg/packages `recipes.txt`. +go-pkgx/packages now redirects to **go-zpkg/packages**. + +| runtime | pantry project | go-zpkg `recipes.txt` | how it is built | note | +|---|---|---|---|---| +| Julia | `julialang.org` (also `julialang.org/juliaup`) | line 1216 | **vendored official binary tarball** (`warnings: vendored`, `curl … julialang-s3…/julia-$v-$TRIPLE.tar.gz`) | not built from source: see open question Q2. Latest release v1.13.3 (2026-10-26) | +| R | `r-project.org` | line 1600 | built from source; gfortran 14 runtime; **bundled reference BLAS** (`no --with-blas`) | platforms `linux` and `darwin/aarch64` only | +| GCC / gfortran | `gnu.org/gcc` (test files include `test.f90`) | line 1043 | from source | gives gfortran and g++ | +| Boost | `boost.org` | line 77 | from source; Math is header-only | the latest GitHub release is `boost-1.93.0.beta1` (2026-11-11): **pin a non-beta** | +| MPFR | `gnu.org/mpfr` | line 1071 | | oracle only | +| FLINT (contains Arb since 3.0) | `flintlib.org` | line 418 | | **LGPL-3.0** (`gh api …/flint/license`): oracle only, black box | +| GSL | `gnu.org/gsl` | line 1056 | | GPL: black box | +| FFTW, OpenBLAS, LAPACK | present (414, 1461, 1416) | | | not needed by go-numerics | + +### 2.2 `special` + +What each Julia function actually calls was read in SpecialFunctions.jl +v2.9.2 (MIT, `Project.toml`; the deps are OpenLibm_jll and OpenSpecFun_jll). +This decides which reference is worth timing. + +| Go function(s) | reference | license | what it is | its accuracy claim (opened) | +|---|---|---|---|---| +| `Jv, Yv, Iv, Kv` (real order), `Jn, Yn, In, Kn`, `Ive, Kve` | **Bessels.jl** v0.2.8 (2023-01-28; master still active) | MIT (`gh api` SPDX) → source | pure Julia, no external dependencies | README: relative error vs ArbNumerics at 500 bits on 1000 points in (0, 100). The maxima are 2e-11 for `besselj(5,x)`, 5e-12 for `bessely(6.92,x)` and 7e-14 for `besseli(92.12,x)`. It claims **5-10× faster** than SpecialFunctions.jl: `besselj(nu,x)` 6×, `bessely` 5×, `besseli` 7×, `besselk` 4×, `besseli0`/`besselk0` 10× (Julia 1.7.2, M1 under Rosetta) | +| same | **SpecialFunctions.jl** `besselj/y/i/k(nu::Real, x)` | MIT; it wraps **AMOS** (`src/bessel.jl:234-303` ccall `zbesh_/zbesi_/zbesj_/zbesk_/zbesy_` from libopenspecfun) | AMOS | openspecfun `LICENSE.md`: AMOS is public domain through SLATEC. Faddeeva is MIT (MIT 2012) | +| `J0, J1, Y0, Y1, Jn, Yn` (integer) | SpecialFunctions.jl → **OpenLibm** `j0/j1/y0/y1/jn/yn` (`src/bessel.jl:212-219,416,440`) | OpenLibm: MIT / ISC (Moshier) / BSD-2 (msun) / FDLIBM. **Its tests `test-double.c` and `test-float.c` are LGPL: do not open them** | msun | none stated | +| Bessel J, Y, I, K (all) | **Boost.Math** `cyl_bessel_j`, `cyl_neumann`, `cyl_bessel_i`, `cyl_bessel_k` | BSL-1.0 → source | Steed's method, Temme series, A&S 9.2.19 asymptotics, Cody minimax rationals for orders 0 and 1 | Table 8.41, double, GCC 7.1 Linux. J general: max 10 ε, mean 2.24. "J0 tricky cases": 8e4 ε. GSL 2.1 and Rmath 3.2.3 on the same data are far worse (J general: 2.4e5 ε; JN: +INF) | +| same, real-argument sequence | **AMOS / SLATEC** Fortran `ZBESJ/ZBESI/ZBESK/ZBESY`, plus SLATEC real-order `DBESJ/DBESI` | public domain (SLATEC guide §4; the SLATEC TOC lists ZBES*-C and DBESJ/DBESI-D) | the routines SciPy and SpecialFunctions.jl wrap | Amos, ACM TOMS 12 (1986), the paper | +| `Gamma` | SpecialFunctions.jl `gamma(::Float64)` = **OpenLibm `tgamma`** (`src/gamma.jl:602`). **Bessels.gamma** (a re-export of Gamma.jl) | MIT | | Bessels.jl README: `Bessels.gamma` max relative error 5e-16, **5× faster** than SpecialFunctions.jl | +| `Gamma, LogGamma, Digamma, Beta` | Boost `tgamma`, `lgamma`, `digamma`, `beta` | BSL | | per-function tables in the Boost docs (only `gamma_p`/`gamma_q` and `cyl_bessel_j` were opened here) | +| `GammaP, GammaQ, GammaPInv, GammaQInv` | **Boost** `gamma_p`, `gamma_q`, `gamma_p_inv` | BSL | | Tables 8.9 and 8.10, double, GCC 7.1 Linux. `gamma_p` max 0.955 ε (medium), 0 ε elsewhere. GSL 2.1: 342 to 1.02e3 ε. Rmath 3.2.3: 389 to 1.11e3 ε. `gamma_q`: GSL "small" 1.38e10 ε | +| same | SpecialFunctions.jl `gamma_inc` (native Julia; DiDonato-Morris; the `IND` flag trades accuracy for speed, `gamma_inc.jl:849` per special's roadmap) | MIT | | | +| `BetaInc, BetaIncC, BetaIncInv, BetaIncCInv, ErfInv` | Boost `ibeta`, `ibetac`, `ibeta_inv`, `ibetac_inv`, `erf_inv` | BSL | **This is already what SciPy calls.** scipy `special/boost_special_functions.h:118-411` wraps these with `promote_double` (`:18-22`) | so the current "BetaInc 2.1× slower than SciPy" row is already a comparison with Boost without promotion | +| `Erf, Erfc` | SpecialFunctions.jl → OpenLibm `erf/erfc` (`src/erf.jl:27`) | as OpenLibm | | | +| all of the above | **R** base: `gamma, lgamma, digamma, beta, besselJ/Y/I/K(…, expon.scaled)`, `pgamma, qgamma, pbeta, qbeta, pnorm, qnorm` | **GPL: black box only** | R docs: Bessel = "a translation of Fortran routines from netlib specfun" (W. J. Cody), C by M. Maechler | R docs: "warnings about accuracy loss for large arguments"; for very large orders "rarely useful". The Boost tables above measured Rmath 3.2.3 | +| same | GSL `gsl_sf_*` | **GPL: black box only** | | Boost tables above | +| oracle | mpmath 1.4.1 (stays canonical, 192/320 bits, double-double `hi+lo`; `testdata/gen/common.py:1-15`) | BSD | | | +| oracle, second opinion | MPFR (LGPL, verify), FLINT/Arb (LGPL-3.0) | **black box** | for regions where mpmath is slow (huge Bessel orders) | | + +Cephes stays a source through SciPy (BSD; attribution in special `README.md` +"License"). + +### 2.3 `integrate` + +| Go | reference | license | how timed | accuracy / work claim | +|---|---|---|---|---| +| `Quad` (QAGS/QAGI/QAWO/QAWC/QAWS) | **QUADPACK Fortran** (netlib `quadpack/readme`: Piessens, de Doncker, Ueberhuber, Kahaner), compiled with gfortran, callbacks in Fortran or C | netlib `readme` states no license. SLATEC (public domain, guide §4) includes it (TOC: QPDOC, DQAGS-D, DQAGI-D, DQK21-D), so take the **SLATEC copy** | compiled worker, §3.3 | the port claims the same subdivision path (integrate README: 116/124 fixtures). This is the purest "cost of the port" comparison: same algorithm, same integrand calls | +| `Quad` | **QuadGK.jl** `quadgk` | MIT | Julia worker, closure integrand (compiled, no interpreter in the loop) | README: returns `(integral, err)`; any order; "convergence was very rapid" on smooth integrands. Report `quadgk_count`'s evaluations as work | +| `Quad` | Boost `gauss_kronrod`, `tanh_sinh` | BSL | compiled worker | not opened: verify | +| `Quad` | R `stats::integrate` | GPL: black box | R worker. The integrand is an R closure **interpreted** per vector call. R requires `f` to be vectorised (docs) | R docs: "Based on QUADPACK routines dqags and dqagi"; `rel.tol` defaults to `eps^0.25`. The row must state that it is not a compiled-callback comparison | +| `SolveIVP` RK45/DOP853/Radau/BDF/LSODA | **OrdinaryDiffEq.jl** DP5, DP8, RadauIIA5, FBDF (Tsit5 and Vern as information) | MIT (Expat) | Julia worker, in-place RHS, `save_everystep=false`, controller set explicitly | compare nf, naccept and nreject, as integrate's roadmap already plans (`docs/roadmap.md` rows 1, 3, 8, 10) | +| `SolveIVP` DOP853/Radau | Hairer `dop853.f`, `radau5.f` | **verify**: unige.ch blocked the fetch (Q6) | compiled worker | | +| `Odeint`/LSODA | ODEPACK `dlsoda` | public domain per integrate `docs/roadmap.md` row 2: **verify on netlib** | compiled worker | nst, nfe, nje, mused | +| `SolveIVP` | R `deSolve` | GPL: black box | R worker, R closure RHS (interpreted) | | + +### 2.4 `optimize` + +| Go | reference | license | how timed | claim | +|---|---|---|---|---| +| `Minimize` BFGS, L-BFGS, CG, NM | **Optim.jl** | MIT | Julia worker, closures, `Optim.Options(...)` matched to SciPy's tolerances | counts in `Optim.f_calls/g_calls` | +| `Minimize` L-BFGS-B | **L-BFGS-B 3.0** (Fortran) or its C translation `stephenbeckr/L-BFGS-B-C` | the C translation is BSD-3-Clause (SPDX opened); the Fortran 3.0 license must be **verified** | compiled worker | | +| `Root`/`Fsolve`, `LeastSquares(LM)`, `Leastsq` | **MINPACK** `hybrd/hybrj/lmder/lmdif` | netlib `minpack/disclaimer`: BSD-like **with an acknowledgment clause** ("This product includes software developed by the University of Chicago, as Operator of Argonne National Laboratory"): keep the notice next to the vendored copy | compiled worker | | +| `CurveFit` | **LsqFit.jl** `curve_fit` | MIT | Julia worker | | +| `Brentq` etc. | **Roots.jl** `find_zero(…, Brent()/A42())` | MIT | Julia worker | | +| `Brentq` | Boost `tools::toms748_solve`, `brent_find_minima` | BSL | compiled worker | not opened: verify | +| `Minimize` NM/BFGS/CG/L-BFGS-B | R `stats::optim` | GPL: black box | R worker, interpreted `fn`/`gr` | R docs: Nelder-Mead, BFGS and CG from Nash (1990) Pascal via p2c; L-BFGS-B "based on Fortran code by Zhu, Byrd, Lu-Chen and Nocedal" from Netlib; `reltol = sqrt(eps)`, `maxit` 100 (500 for NM); `counts` = fn and gr calls | + +For `optimize` and `integrate` a time is only meaningful together with the +**work** (function evaluations) and the **error** against the exact answer. +A faster QuadGK that uses a different number of evaluations measures the +algorithm, not the port. Both are reported (§4). + +## 3. refbench extension + +### 3.1 Several references per case + +```go +type Ref struct { + ID string // stable key, ":", e.g. "julia:Bessels.besseli", "boost:cyl_bessel_i", "fortran:amos.zbesi" + Name string // human label for the table + License string // SPDX or "GPL-black-box"; ReadReport refuses an empty one + Timer Timer + Eval EvalFunc // nil: no accuracy column for this reference +} + +type Case struct { + // ... unchanged fields ... + Ref Timer // primary reference (SciPy today): unchanged semantics + Refs []Ref // additional references; may be empty + // Accuracy, when set, scores Go and every Ref with Eval on the same points (§4). + Accuracy *Accuracy +} +``` + +- **The primary reference stays** (`Reference`, `Ref`, `ratio`, `verdict` keep + their meaning). Existing tables, issue titles and README sentences ("vs SciPy + 1.18.1") remain true. Additional references are additional columns. +- **Round order.** With k = 1 + 1 + len(Refs) sides, round r runs the sides + in order rotated by r mod k. Every side therefore appears in every + position once per k rounds, and the default number of rounds is rounded up + to a multiple of k. For k = 2 this is exactly today's alternation. (A plain + rotation balances positions, not "who runs just before whom". For k > 2 a + balanced Latin square would also balance that; it is left as a refinement + because the cache effect between processes is small.) +- **Per-reference validity.** A round that ends above the gate is discarded + for all its sides (today's rule). Retries stay per row. `Options.Refs` + (flag `-perf.refs=julia,boost`) restricts a run to some kinds, so a long + multi-reference row can be measured later. The deferred list carries + `case` or `case@refID`, and `Merge` merges per (case, ref). + +### 3.2 Verdict: against each reference, headline against the best + +- One verdict **per reference**, with today's rule (`Judge`: quartile + overlap, ratio Go/ref). +- A derived **`best`**: the reference with the lowest median among those + whose accuracy was scored. The row's headline is the verdict against + `best`. +- **An accuracy-aware label**, so that "faster than Julia but less accurate" + is never counted as a win and "slower than a less accurate code" is not + hidden: + + | time vs ref | max ulp, Go vs ref | label | + |---|---|---| + | faster or tie | Go ≤ ref | `ahead` | + | faster or tie | Go > ref | `faster-less-accurate` (an **accuracy** defect) | + | slower | ref ≤ Go's budget | `behind` (a **perf** defect, top priority) | + | slower | ref > Go's budget | `tradeoff` (still a perf defect under the fleet rule, labelled so) | + +- `refbench issues` drafts **one issue per case** (not one per reference), + titled after `best`: `perf: is x slower than on one core`. + The body lists every reference with its ratio, verdict, max ulp, license + and version, and carries the label above. Drafting one issue per + reference would triple the tracker noise for a single defect. + +### 3.3 Workers + +One protocol for the new kinds, plain text, because R base and Fortran have +no JSON parser and adding one (jsonlite, json-fortran) is one more pinned +dependency. Python keeps its JSON protocol unchanged, so pinned repos are not +broken. The Go side hides both behind one interface. + +``` +rbw1 protocol, one request line → one reply line, ASCII, ≤ 4 KiB per line +L load float64 LE column-major args → "ok " +W one untimed call (JIT warm-up, page-in) → "ok" +T time iters calls in-process → "ns " +E evaluate once per point, write float64 → "ok " +V versions and observed threading → "v k=v k=v …" +error on any line → "err "; the runner stops (a series with a hole is biased) +``` + +- `fn` is a **name in a fixed table** compiled or defined in the worker + (`refs/julia/kernels.jl`, `refs/r/kernels.R`, `refs/boost/kernels.cpp`). + Nothing is eval'd from the pipe. Paths must lie inside the run directory. + `cols*rows*8` is bounded (refuse > 256 MiB). Line length is bounded. The + parser gets a fuzz target and a committed corpus, as `FuzzParseReply` has. +- `Calibrate` additionally discards one first call (`t(1)`, untimed). This + fixes the JIT problem in §1 for every kind, and costs one call. + +| kind | launch (all under `SingleThreadEnv()` plus the extras) | timed body | versions reported (`V`) | single-thread extras, **verified from `V`, not trusted** | +|---|---|---|---|---| +| python | unchanged (`uv run --python 3.12.12 …`) | unchanged | unchanged | unchanged | +| julia | `pkgx +julialang.org@= -- julia --startup-file=no --history-file=no -t 1 --project=/refs/julia ` with `JULIA_DEPOT_PATH=/refbench/julia-depot`, `JULIA_PKG_OFFLINE=true` | `out .= f.(x)` inside a function barrier on a typed `Vector{Float64}`, timed with `time_ns()`. No `@fastmath`, default `--math-mode`. `@allocated` reported as information | `VERSION`, LLVM version, `Sys.CPU_NAME`, every package in `Manifest.toml` with version and git-tree-sha1, `BLAS.get_config()`, `Threads.nthreads()`, `BLAS.get_num_threads()` | `JULIA_NUM_THREADS=1`, `JULIA_NUM_GC_THREADS=1` (Julia ≥ 1.10, docs), `-t 1`, `BLAS.set_num_threads(1)` at start. **Refuse** if `V` reports nthreads ≠ 1 | +| r | `pkgx +r-project.org@= -- Rscript --vanilla ` | one vectorised call (`besselI(x, nu, expon.scaled=…)`, `pgamma(x, a)`) per iteration; R allocates its result. Clock: `Sys.time()` differences over ≥ 100 ms samples | `R.version.string`, `La_library()`, `extSoftVersion()` | bundled reference BLAS (pkgx build: no `--with-blas`); OMP=1 already set | +| compiled (C, C++, Fortran) | binary built by `BuildWorker` (a generalised `BuildC`): pkgx-pinned gcc/g++/gfortran and pinned Boost headers; sources committed with SHA-256 | a C/Fortran loop over the array writing `out[i]`, with a checksum returned so the loop cannot be removed | compiler `__VERSION__`, flags, `BOOST_VERSION`, Boost policy, `LDBL_MANT_DIG`, the SHA-256 of every source | no OpenMP, no threads | + +**Compiler flags of compiled references.** For speed the fair target is the +reference's normal release build, not our parity build. The default is +`-O2`, with FP contraction at the compiler default and no `-march=native`. +The flags are recorded, and a different flag set is a different `Ref.ID`. +`NoFMAFlags` stays for parity rows only. + +**Boost policy.** Boost's default is `promote_double`, which computes +`double` in `long double` (Boost docs, "Internal Floating-point Promotion +Policies"). SciPy turns it off. On arm64 macOS `long double` is `double`, so +the two policies are the same build. On x86-64 promotion means 80-bit x87. +On aarch64 Linux it means 128-bit software quad, which would be very slow. +So the design registers `boost` (promote off, SciPy's choice, the speed +reference) and, on x86-64 only, `boost-promote` (the accuracy reference that +gives Boost's 0 ε tables). `LDBL_MANT_DIG` goes in the record. + +**Where worker sources live.** These include the embedded `worker.jl`, +`worker.R` and the rbw1 C/Fortran harness, which are generic. They also +include the per-library kernels: Boost special functions, AMOS, QUADPACK, +MINPACK and L-BFGS-B. The generic part goes in refbench, embedded like +`pyworker.py`. The library kernels go in **one shared place, not per +repository** (Q1). Only permissive sources are ever vendored. GPL references +are reached only through their installed binaries (R) or their documented +API (GSL, if ever). + +### 3.4 Data crosses languages as bytes + +- The Go side writes each case's argument arrays once per run, as float64 + little-endian column-major, into + `os.UserCacheDir()/refbench///.f64`. That directory + belongs to the tool, not to the repository and not to `/tmp`. +- `L` returns the SHA-256 of the bytes the worker read. The runner compares + it with the Go side's digest before any timing, and refuses on a mismatch, + naming both digests. This replaces `DataHash` for the new kinds and + generalises `TestDataParity`. +- The Python worker gains the same `load` request (read the file with + `np.fromfile`), so all sides time exactly the same bytes. The NumPy + `setup` expression stays for compatibility. + +### 3.5 Pinned versions in the record + +- `ref_versions` stays a flat map with **namespaced keys**: + `julia/julia=1.13.3`, `julia/Bessels=0.2.8@`, + `boost/version=1.92.0`, `boost/policy=promote_double:false`, + `fortran/gfortran=…`, `fortran/src/amos=sha256:…`, `r/R=…`. This is no + schema change. `CheckResume` already refuses differing reference + versions, so it now covers every kind. +- Pins live in the repository: `bench/refs.txt` (uv, hashes; unchanged), + `bench/refs/julia/Project.toml` and `Manifest.toml` (exact versions and + git-tree-sha1 content hashes, Julia's equivalent of `--generate-hashes`), + `bench/refs/toolchain.txt` (`julialang.org@=1.13.3`, `r-project.org@=…`, + `gnu.org/gcc@=…`, `boost.org@=…`). +- `refbench prepare` (new subcommand) instantiates and precompiles the Julia + project and builds the compiled workers, **outside** any gated run. + Precompilation is minutes of CPU and must not happen inside a timed + window, nor push the load over the gate during a run. + +### 3.6 JSON compatibility: exactly what breaks + +New fields: + +- On `Row`: `refs: [{id, name, license, iters, summary, ratio, verdict, label, accuracy}]`, + `go_accuracy`, `best`. +- On `Round`: `refs_ns: {id: ns}` and `order: [side ids]`. + +`Round.ref_ns`, `Row.ratio` and `Row.verdict` keep the primary reference's +meaning. + +- **A repo with no `Refs` keeps writing schema 1, byte for byte.** Nothing + changes for repositories that do not opt in. +- **A repo with `Refs` writes schema 2.** The new `ReadReport` reads 1 and 2 + (1 → no extra refs). +- **Break:** an older refbench (any build with `SchemaVersion = 1` and + `DisallowUnknownFields`) refuses a schema-2 file with + `perf.json: schema 2, this build reads 1`. That covers the `rbtest` in a + repo's `bench/go.mod` and the installed `refbench table/check/issues`. A + repository that adds `Refs` must bump refbench in the same PR. The refusal + is loud, never silent. +- The table gains a second section, "Against other references" (§4). The + first section is unchanged, so `refbench check` on a schema-1 file is + unchanged. + +### 3.7 Staying cheap in CI + +- CI mode still starts no runtime and times nothing. It additionally checks + four things: + - every `Ref.ID` declared in the registry has a row entry, or is in + `deferred`; + - every ref has a non-empty `license`; + - no ref is labelled GPL unless its kind is a black-box kind (`r`, `gsl`); + - the table regenerates from the JSON. +- refbench's own tests use fake workers through the replaceable + `execCommand` (today's pattern). Coverage stays at 100 %. Fuzz targets for + the rbw1 reply parser and the schema-2 reader are added. +- An optional, manually dispatched lane (weekly at most) installs Julia and + R through pkgx and runs only the protocol smoke test (`V`, `L`, `W`, one + `E`). It never times. + +## 4. Accuracy in the same rows + +The fleet contract is "a stated bound against an extended-precision oracle". +The same oracle must judge every reference. Judge and subject must count +alike: the same points, the same ulp function, the same treatment of +non-finite values. + +```go +type Accuracy struct { + Points [][]float64 // columns; written once, loaded with L + Score func(got []float64) AccResult // the repo's own scorer (special: ulp vs mpmath hi+lo) +} +type AccResult struct { + MaxULP float64 // +Inf if a finite value was expected and NaN/Inf returned + MaxRegion string // where the max occurs + MaxULPFin float64 // max over points where the reference returned a finite value + Coverage float64 // fraction of points the reference evaluated (exceptions → NaN, counted) + Points int +} +``` + +- **special.** `Points` are the fixture arguments of the function. The + fixtures are `testdata/fixtures/.tsv.gz` with columns region, args, + mpmath hi, mpmath lo, scipy (`testdata/gen/common.py:146`). `Score` is the + existing ulp measure of `TestAccuracyBound`. Go's own column comes from + the same `Score` applied to the Go function on the same points. +- **The first run must reproduce the fixtures' SciPy column bit for bit** + through the Python worker's `E`. That is the negative control that the + eval path passes the arguments unaltered (a planted perturbation of one + argument must make it fail). +- **Domain conventions differ.** Bessels.jl throws `DomainError` for a + negative x with a complex result (its README), R warns and returns NaN, and + Boost raises according to its policy. Workers catch per point, return NaN + and count it (`n_raised`). Hence the two columns `MaxULP` (honest + worst case) and `MaxULPFin` with `Coverage`, so that a reference which + refuses half the domain does not look accurate. +- **optimize and integrate.** `E` returns per problem `(value, nfev[, njev])`. + `Score` gives the error against the exact optimum or integral (the + existing `math/big` and mpmath oracles in `accuracy_bound_test.go` and + integrate's fixtures). The table shows time, error and evaluations side by + side. +- Accuracy is deterministic, so it is computed **once per (case, ref, + versions, machine)**, ungated, before the timing rounds, and stored in the + record. `-perf.resume` does not recompute it unless the versions changed. +- Table, second section: + + `| case | Go ns | Go max ulp | best | ratio vs best | label | julia:… ratio (max ulp, cov) | boost:… | r:… |` + +## 5. Priorities + +The expected value is the likely gap, times how often the function is used, +times whether the reference is also a legal algorithm source. The cost +assumes the core (§3.1-3.6) and **compiled workers** first: gfortran is +present and Boost is header-only. Then the **Julia worker**, then **R**. + +| # | comparison | reference (license) | why | expected | cost | +|---|---|---|---|---|---| +| 1 | `Iv, In, Ive, Kv, Kn, Kve` | **Bessels.jl** (MIT, source) + Boost `cyl_bessel_i/k` (BSL) | Go is already 8.0× (Iv), 5.3× (In), 2.3× (Kv), 1.88× (Kn), 1.71× (Ive) slower than SciPy (special README "Speed"). SpecialFunctions.jl routes these to AMOS (`bessel.jl:251,285`), and Bessels.jl claims 7× (`besseli`) and 4× (`besselk`) over it, with max relative error ≤ 7e-14 for `besseli`. **The gap against Julia may be 15-50×**, and Bessels.jl is MIT, so the algorithms can be ported | very high | Julia worker (M) | +| 2 | `Jv, Yv, Jn, Yn` | **Bessels.jl** (MIT) + Boost `cyl_bessel_j/cyl_neumann` (BSL) | Yn is 33.4× slower than SciPy, Jn 1.52×, Jv 1.28×, Yv 1.22×. Bessels.jl claims 6× and 5× over AMOS but up to 2e-11 relative near zeros, where Go holds 0.5-1.7 ulp: likely a `tradeoff`, which is exactly what the accuracy column is for. Boost reaches 10 ε max on J general (Table 8.41) | very high | shares #1 | +| 3 | `Gamma, LogGamma, Digamma, Beta` | Bessels.gamma / Gamma.jl (MIT), OpenLibm `tgamma` via SpecialFunctions.jl, Boost `tgamma/lgamma/digamma` (BSL, both policies on x86-64) | Go beats SciPy (Gamma 0.83, 8.14 ns/element), but Bessels.jl claims `gamma` is 5× faster than OpenLibm's `tgamma` at 5e-16. Gamma is the most-called kernel of the stack (every distribution) | high | cheap once #1 exists; Boost via compiled worker (S) | +| 4 | `GammaP, GammaQ, GammaPInv, GammaQInv` | **Boost** `gamma_p/gamma_q/gamma_p_inv` (BSL); SpecialFunctions.jl `gamma_inc` (MIT) | Go is 1.15× (P/Q) and 1.5× (inverses) slower than SciPy. Boost tables: ≤ 0.955 ε against GSL/Rmath at 10²-10³ ε. SciPy routes only `gamma_p_inva`, `gamma_q_inva`, `lgamma_p` and `lgamma_q` to Boost (`boost_special_functions.h:2566-2757`), so Boost `gamma_p` is a **new** data point | high | compiled worker (S) | +| 5 | `Quad` (QAGS, QAGI, weighted) | **QUADPACK Fortran** from SLATEC (public domain), gfortran `-O2`, Fortran or C callbacks; then QuadGK.jl (MIT) | The README says 0.08-0.29 of SciPy's time even with compiled callbacks. The same Fortran, compiled, is the honest test of what a pure-Go port costs (same path, same evaluations). It also checks that claim | medium-high (validates or corrects a published claim) | compiled worker (S) | +| 6 | `Minimize` BFGS / L-BFGS-B, `Brentq` | **Optim.jl** (MIT), L-BFGS-B-C (BSD-3), Roots.jl (MIT) | "330× faster than SciPy" on Rosenbrock n=2 is mostly Python callback overhead (optimize README says so). Compiled-callback references give the number that will survive scrutiny | medium-high | Julia worker (shared) + compiled (S) | +| 7 | `Root`, `LeastSquares(LM)`, `Leastsq`, `CurveFit` | **MINPACK** (BSD-like + acknowledgment) gfortran; LsqFit.jl (MIT) | the Go MINPACK port with FMA/go-simd above 32 residuals against the original Fortran: a pure kernel-speed comparison | medium | compiled (S) | +| 8 | `SolveIVP` RK45/DOP853/Radau/BDF | **OrdinaryDiffEq.jl** DP5/DP8/RadauIIA5/FBDF (MIT); Hairer Fortran (license Q6) | Julia's ODE suite is the domain's performance reference. Counts (nf, steps) are already on integrate's roadmap, so the timings complete that picture | medium | Julia (OrdinaryDiffEq precompile is heavy: `refbench prepare`) (M) | + +Below the top 8: + +- **BetaInc/BetaIncInv/ErfInv vs Boost.** SciPy already *is* Boost without + promotion (§2.2), so only `boost-promote` (x86-64) adds information, as + accuracy. +- **Erf/Erfc vs OpenLibm** (through Julia). +- **AMOS compiled directly**: redundant with SpecialFunctions.jl, and cheap + once the Fortran harness exists. +- **R** for all functions, black box: low speed value (Cody/Nash code, and + interpreted callbacks for integrate and optim), but it is "what statisticians + run". It is mostly useful for go-statistics later. +- **GSL**, black box: Boost's tables already show its accuracy. + +Order of work: + +1. Core: Refs, rounds, schema 2, `Accuracy`, rbw1, `Calibrate` warm-up, byte data. +2. Compiled worker, with Boost (#3, #4) and QUADPACK/MINPACK (#5, #7). +3. Julia worker (#1, #2, #6, #8). +4. R worker. + +Each step ships with its README, landing and docs updates, per the fleet +rule. + +## 6. Open questions for the user + +- **Q1. Home of the per-library kernels** (Boost, AMOS, QUADPACK, MINPACK, + L-BFGS-B). The options are inside refbench (`workers/`), a new shared repo + (`go-scikit/refworkers`), or each repo's `bench/refs/`. Recommendation: a + shared repo, so there is one implementation for every consumer. +- **Q2. Julia comes from pkgx as a vendored upstream binary** (`julialang.org` + `package.yml`: `warnings: vendored`), not built by go-zpkg. Is that + acceptable under the sovereign-build doctrine for a *measurement* + reference, or must go-zpkg build Julia from source first? +- **Q3. GSL and Rmath as black boxes.** Is it acceptable to write our own + worker that links them, if we never read their code or headers, only the + manuals, and never distribute the binary? Or is it R through `Rscript` + only? +- **Q4. Issue policy.** Should a case slower than a *less accurate* + reference (`tradeoff`) be drafted as a perf defect? The fleet rule says + yes. The design drafts it with the label. +- **Q5. Boost promotion.** Should `boost-promote` be measured on the x86-64 + host only? On the reference host (Apple M4 Max) the two policies are the + same build. +- **Q6. Licences not settled by an opened document:** + - Hairer DOP853/RADAU5: the site blocked the fetch; + - L-BFGS-B 3.0 Fortran (the C translation is BSD-3); + - ODEPACK; + - Cody's netlib SPECFUN (the code behind R's Bessel functions, which would + give R's algorithm without GPL); + - MPFR. +- **Q7. Using Bessels.jl (MIT) as an algorithm source** to close gaps #1 and + #2. Measuring it is this design. Porting it is the follow-up the + measurement will probably call for. +- **Q8. Primary reference.** Recommendation: keep SciPy as the primary, so + that existing tables and READMEs stay true. Alternatively make the primary + the best reference. + +## 7. Sources opened for this design + +- go-scikit/refbench `origin/main` 8b6268b: `README.md`, `case.go`, `report.go`, `runner.go`, `worker.go`, `pyworker.py`; PR #10 body. +- go-numerics/special README; `docs/roadmap.md`; local clone `go-numerics-special-ynjn` 8255bbe: `bench/perf_test.go:1-90`, `testdata/gen/common.py:1-40,146`. +- go-numerics/optimize README and `docs/roadmap.md` (headings and rows); go-numerics/integrate README and `docs/roadmap.md` (headings and rows). +- pkgxdev/pantry `projects/julialang.org/package.yml`, `projects/r-project.org/package.yml`, `projects/boost.org/package.yml`, directory listings of `gnu.org/gcc`, `gnu.org/mpfr`, `fftw.org`, `flintlib.org`; go-zpkg/packages `recipes.txt`. +- JuliaMath/SpecialFunctions.jl `LICENSE`, `Project.toml` (v2.9.2), `src/bessel.jl`, `src/erf.jl`, `src/gamma.jl` (ccall lines); JuliaMath/openspecfun `LICENSE.md`, `README.md`; JuliaMath/openlibm `LICENSE.md`; JuliaMath/Bessels.jl README, license, latest release; JuliaMath/QuadGK.jl README and license; JuliaNLSolvers/Optim.jl `LICENSE.md`; JuliaNLSolvers/LsqFit.jl `LICENSE.md`; SciML/OrdinaryDiffEq.jl `LICENSE.md`; JuliaMath/Roots.jl license (SPDX); flintlib/flint license (SPDX); stephenbeckr/L-BFGS-B-C license (SPDX); boostorg/math license (SPDX). +- scipy/scipy `scipy/special/boost_special_functions.h` (policy at :18-32, wrapped functions). +- Boost.Math docs: Bessel first/second kind (Table 8.41), incomplete gamma (Tables 8.9, 8.10), "Internal Floating-point Promotion Policies", "Relative error". +- Julia manual, "Environment Variables" (`JULIA_NUM_THREADS`, `JULIA_NUM_GC_THREADS`, `JULIA_EXCLUSIVE`, `JULIA_CPU_TARGET`, `JULIA_DEPOT_PATH`, `JULIA_PKG_OFFLINE`). +- R manual pages: `base::Bessel`, `stats::integrate`, `stats::optim`. +- netlib: `minpack/disclaimer`, `quadpack/readme`, `slatec/guide` (§4), `slatec/toc` (first 100 kB). +- Not opened (claims above marked "verify"): Hairer's site (blocked), the Boost quadrature and root-finding docs, the MPFR licence, the ODEPACK licence, the L-BFGS-B 3.0 Fortran licence, BenchmarkTools.jl (not used: refbench's own calibration and statistics stay the single source of truth). diff --git a/golden_test.go b/golden_test.go new file mode 100644 index 0000000..8860b45 --- /dev/null +++ b/golden_test.go @@ -0,0 +1,43 @@ +package refbench + +import ( + "bytes" + "flag" + "os" + "path/filepath" + "testing" +) + +var updateGolden = flag.Bool("update-golden", false, "rewrite testdata/golden (only to record a deliberate schema 1 change)") + +// TestGoldenSchema1 pins the bytes a repository with no extra reference writes: +// perf.json and perf-table.md must stay byte for byte what they were before +// several references per case existed (the files in testdata/golden were +// written by the code of v0.3.x, before Case.Refs). +func TestGoldenSchema1(t *testing.T) { + r := sampleReport() + var js bytes.Buffer + if err := r.Write(&js); err != nil { + t.Fatal(err) + } + table := Table(r) + for name, got := range map[string][]byte{"schema1.json": js.Bytes(), "schema1-table.md": table} { + path := filepath.Join("testdata", "golden", name) + if *updateGolden { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("%s differs from its golden file: schema 1 output changed", name) + } + } +} diff --git a/issues.go b/issues.go index 536194c..95edcf6 100644 --- a/issues.go +++ b/issues.go @@ -31,12 +31,22 @@ func Issues(rep *Report) ([]Issue, error) { } var out []Issue for _, row := range rep.Rows { - if !row.Valid || row.Verdict != Slower { + slower := slowerSides(row) + if !row.Valid || len(slower) == 0 { continue } + // One issue per case, titled after the fastest side Go is slower than: + // a gap against ANY faster reference is a defect, even a less accurate + // one (a "tradeoff" is labelled, not excused). + target := slower[0] + for _, s := range slower[1:] { + if s.sum.Median < target.sum.Median { + target = s + } + } out = append(out, Issue{ Case: row.Name, - Title: fmt.Sprintf("perf: %s is %.2fx slower than %s on one core", row.Name, row.Ratio, row.Reference), + Title: fmt.Sprintf("perf: %s is %.2fx slower than %s on one core", row.Name, target.ratio, target.name), Labels: []string{"perf"}, Body: issueBody(rep, row), }) @@ -51,6 +61,20 @@ func issueBody(rep *Report, row Row) string { fmt.Fprintf(&b, "| Go | %.4g | %.4g | %.4g |\n", row.Go.Median, row.Go.Q1, row.Go.Q3) fmt.Fprintf(&b, "| reference | %.4g | %.4g | %.4g |\n\n", row.Ref.Median, row.Ref.Q1, row.Ref.Q3) fmt.Fprintf(&b, "Ratio Go / reference: **%.2f**. Allocations per call (Go): %g.\n\n", row.Ratio, row.Allocs) + if len(row.Refs) > 0 { + b.WriteString("Every reference of the case (ratio = Go / reference; a reference that is faster is a defect even when it is less accurate):\n\n") + b.WriteString("| reference | license | median ns/call | ratio | verdict | label | max ulp | coverage |\n|---|---|---:|---:|---|---|---:|---:|\n") + fmt.Fprintf(&b, "| %s (primary) | | %.4g | %.2f | %s | %s | %s | %s |\n", esc(row.Reference), row.Ref.Median, row.Ratio, row.Verdict, + Label(row.Verdict, row.GoAccuracy, row.PrimaryAccuracy, row.GoBudget), ulpCell(row.PrimaryAccuracy), covCell(row.PrimaryAccuracy)) + for _, ref := range row.Refs { + fmt.Fprintf(&b, "| %s (`%s`) | %s | %.4g | %.2f | %s | %s | %s | %s |\n", esc(ref.label()), ref.ID, ref.License, ref.Summary.Median, ref.Ratio, ref.Verdict, ref.Label, ulpCell(ref.Accuracy), covCell(ref.Accuracy)) + } + fmt.Fprintf(&b, "\nGo: max ulp %s", ulpCell(row.GoAccuracy)) + if row.GoBudget > 0 { + fmt.Fprintf(&b, " (stated bound %g)", row.GoBudget) + } + b.WriteString(". The goal is to be faster without giving up accuracy.\n\n") + } fmt.Fprintf(&b, "- Commit: %s (%s)\n- Date: %s\n- Go: %s\n- Machine: %s\n", rep.Commit, rep.Repo, rep.Date, rep.Go, rep.Machine) keys := make([]string, 0, len(rep.RefVersions)) for k := range rep.RefVersions { diff --git a/multiref.go b/multiref.go new file mode 100644 index 0000000..d280ed5 --- /dev/null +++ b/multiref.go @@ -0,0 +1,433 @@ +package refbench + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "regexp" + "strings" +) + +// SchemaMultiRef is the schema of a perf.json whose rows carry extra +// references. A repository with no Case.Refs keeps writing SchemaVersion (1). +const SchemaMultiRef = 2 + +// Reserved side names in a Round's Order and in Row.Best. +const ( + SideGo = "go" + SidePrimary = "primary" +) + +// EvalFunc evaluates a function once on the case's accuracy points and returns +// one value per point. A point the reference refuses (a domain error) is NaN; +// the worker counts it. +type EvalFunc func() ([]float64, error) + +// Ref is one additional reference of a case. The primary reference stays +// Case.Ref (SciPy today): its tables and sentences keep their meaning. +type Ref struct { + // ID is the stable key ":", e.g. "julia:Bessels.besseli", + // "boost:cyl_bessel_i", "fortran:amos.zbesi". The kind is a worker family. + ID string + // Name is the label of the table; empty means ID. + Name string + // License is an SPDX identifier, or "GPL-black-box" for a reference that is + // only ever run, never read or linked (kinds "r" and "gsl"). + License string + // Timer times the reference inside its own process. + Timer Timer + // Eval scores the reference's accuracy; nil gives it no accuracy column. + Eval EvalFunc +} + +// Accuracy scores Go and every Ref that has an Eval on the same points, with +// the repository's own scorer (special: ulp against the mpmath hi+lo fixtures). +type Accuracy struct { + // Go evaluates the Go function on the points. + Go EvalFunc + // Score returns the error of one evaluation. Judge and subject count alike: + // the same points, the same ulp function, the same treatment of non-finite + // values, for every side. + Score func(got []float64) AccResult + // Budget is the maximum ulp Go promises (its stated bound). It separates a + // "behind" (the reference is as accurate as Go promises) from a "tradeoff" + // (the reference is less accurate than that). Zero means Go's own measured + // maximum. + Budget float64 +} + +// ULP is an error in units in the last place. It is +Inf when a finite value +// was expected and NaN or Inf came back, which JSON cannot carry: it is the +// string "inf" there. +type ULP float64 + +// MarshalJSON encodes a finite value as a number and +Inf as "inf". +func (u ULP) MarshalJSON() ([]byte, error) { + f := float64(u) + switch { + case math.IsInf(f, 1): + return []byte(`"inf"`), nil + case math.IsNaN(f) || math.IsInf(f, -1) || f < 0: + return nil, errors.New("ulp is not a non-negative number or +Inf") + } + return json.Marshal(f) +} + +// UnmarshalJSON reads a number or the string "inf". +func (u *ULP) UnmarshalJSON(b []byte) error { + if string(b) == `"inf"` { + *u = ULP(math.Inf(1)) + return nil + } + var f float64 + if err := json.Unmarshal(b, &f); err != nil { + return fmt.Errorf("ulp: %w", err) + } + *u = ULP(f) + return nil +} + +// AccResult is the accuracy of one side on the case's points. +type AccResult struct { + // MaxULP is the honest worst case: +Inf if a finite value was expected and + // NaN or Inf was returned. + MaxULP ULP `json:"max_ulp"` + // MaxRegion says where the maximum occurs. + MaxRegion string `json:"max_region,omitempty"` + // MaxULPFin is the maximum over the points where the side returned a finite + // value, to be read with Coverage. + MaxULPFin ULP `json:"max_ulp_fin"` + // Coverage is the fraction of points the side evaluated (exceptions count + // as not evaluated), in [0, 1]. + Coverage float64 `json:"coverage"` + // Points is the number of points scored. + Points int `json:"points"` +} + +func (a AccResult) valid() bool { + m, f := float64(a.MaxULP), float64(a.MaxULPFin) + return !math.IsNaN(m) && m >= 0 && !math.IsNaN(f) && f >= 0 && + !math.IsNaN(a.Coverage) && a.Coverage >= 0 && a.Coverage <= 1 && a.Points >= 0 +} + +// RefRow is one extra reference's result in a row. ID, Name and License come +// from the registry; the rest is derived from the rounds by Finish. +type RefRow struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + License string `json:"license"` + Iters int `json:"iters,omitempty"` + // Summary, Ratio, Verdict and Label: Go against this reference. + Summary Summary `json:"summary"` + Ratio float64 `json:"ratio,omitempty"` + Verdict string `json:"verdict,omitempty"` + Label string `json:"label,omitempty"` + Accuracy *AccResult `json:"accuracy,omitempty"` +} + +// Labels of a side against Go (design, section 3.2). +const ( + // LabelAhead: Go is faster or ties, and is at least as accurate. + LabelAhead = "ahead" + // LabelFasterLessAccurate: Go is faster or ties, and is less accurate: an + // accuracy defect. + LabelFasterLessAccurate = "faster-less-accurate" + // LabelBehind: Go is slower than a reference that is as accurate as Go + // promises: a perf defect. + LabelBehind = "behind" + // LabelTradeoff: Go is slower than a reference that is less accurate than + // Go promises: still a perf defect (the fleet rule), labelled so. + LabelTradeoff = "tradeoff" +) + +// Label combines the time verdict with the accuracy. Without accuracy on both +// sides a faster Go has no label (no accuracy claim is made) and a slower Go is +// behind. +func Label(verdict string, goAcc, refAcc *AccResult, budget float64) string { + if verdict == Slower { + if goAcc != nil && refAcc != nil { + if budget <= 0 { + budget = float64(goAcc.MaxULP) + } + if float64(refAcc.MaxULP) > budget { + return LabelTradeoff + } + } + return LabelBehind + } + if verdict == "" || goAcc == nil || refAcc == nil { + return "" + } + if goAcc.MaxULP <= refAcc.MaxULP { + return LabelAhead + } + return LabelFasterLessAccurate +} + +var refIDPattern = regexp.MustCompile(`^[a-z][a-z0-9-]*:[A-Za-z0-9][A-Za-z0-9_.+-]*$`) + +// RefKind returns the kind of a reference ID ("julia" for "julia:Bessels.besseli"). +func RefKind(id string) string { + k, _, _ := strings.Cut(id, ":") + return k +} + +// blackBoxKinds are the kinds whose code is GPL: run, never read or linked. +var blackBoxKinds = map[string]bool{"r": true, "gsl": true} + +// validateRefs checks the registry's extra references of one case. +func validateRefs(c Case) error { + seen := map[string]bool{} + for _, r := range c.Refs { + switch { + case !refIDPattern.MatchString(r.ID): + return fmt.Errorf("case %q: reference ID %q is not :", c.Name, r.ID) + case seen[r.ID]: + return fmt.Errorf("case %q: reference %q is registered twice", c.Name, r.ID) + case strings.TrimSpace(r.License) == "": + return fmt.Errorf("case %q: reference %q has no license", c.Name, r.ID) + case strings.Contains(strings.ToUpper(r.License), "GPL") && !blackBoxKinds[RefKind(r.ID)]: + return fmt.Errorf("case %q: reference %q is %s but its kind %q is not a black-box kind (r, gsl)", c.Name, r.ID, r.License, RefKind(r.ID)) + } + seen[r.ID] = true + } + if c.Accuracy != nil && (c.Accuracy.Go == nil || c.Accuracy.Score == nil) { + return fmt.Errorf("case %q: Accuracy needs Go and Score", c.Name) + } + if c.Accuracy != nil && (math.IsNaN(c.Accuracy.Budget) || c.Accuracy.Budget < 0) { + return fmt.Errorf("case %q: Accuracy.Budget is negative or NaN", c.Name) + } + return nil +} + +func (c Case) refRows() []RefRow { + if len(c.Refs) == 0 { + return nil + } + out := make([]RefRow, len(c.Refs)) + for i, r := range c.Refs { + name := r.Name + if name == r.ID { + name = "" + } + out[i] = RefRow{ID: r.ID, Name: name, License: r.License} + } + return out +} + +// sideLabel is the table label of a RefRow. +func (r RefRow) label() string { + if r.Name != "" { + return r.Name + } + return r.ID +} + +// usesRefs reports whether any row carries extra references. +func (r *Report) usesRefs() bool { + for _, row := range r.Rows { + if len(row.Refs) > 0 { + return true + } + } + return false +} + +// finishRefs derives the per-reference summaries, ratios, verdicts, labels and +// the headline (the fastest side, primary included) of one row from its rounds. +func finishRefs(row *Row) { + row.Best, row.BestRatio, row.BestVerdict, row.BestLabel = "", 0, "", "" + for i := range row.Refs { + ref := &row.Refs[i] + ref.Summary, ref.Ratio, ref.Verdict, ref.Label = Summary{}, 0, "", "" + var xs []float64 + for _, rd := range row.Rounds { + xs = append(xs, rd.RefsNs[ref.ID]) + } + if len(xs) == 0 { + continue + } + ref.Summary = Summarize(xs) + ref.Ratio = row.Go.Median / ref.Summary.Median + ref.Verdict = Judge(row.Go, ref.Summary) + ref.Label = Label(ref.Verdict, row.GoAccuracy, ref.Accuracy, row.GoBudget) + } + if len(row.Refs) == 0 || len(row.Rounds) == 0 { + return + } + best, bestSum, bestAcc := SidePrimary, row.Ref, row.PrimaryAccuracy + for _, ref := range row.Refs { + if ref.Summary.Median < bestSum.Median { + best, bestSum, bestAcc = ref.ID, ref.Summary, ref.Accuracy + } + } + row.Best = best + row.BestRatio = row.Go.Median / bestSum.Median + row.BestVerdict = Judge(row.Go, bestSum) + row.BestLabel = Label(row.BestVerdict, row.GoAccuracy, bestAcc, row.GoBudget) +} + +// slowerSides returns, in the report's order, the label and the summary of +// every side (primary included) that Go is slower than. +type side struct { + id, name string + sum Summary + ratio float64 + label string + acc *AccResult +} + +func slowerSides(row Row) []side { + var out []side + if row.Verdict == Slower { + out = append(out, side{SidePrimary, row.Reference, row.Ref, row.Ratio, Label(Slower, row.GoAccuracy, row.PrimaryAccuracy, row.GoBudget), row.PrimaryAccuracy}) + } + for _, r := range row.Refs { + if r.Verdict == Slower { + out = append(out, side{r.ID, r.label(), r.Summary, r.Ratio, r.Label, r.Accuracy}) + } + } + return out +} + +// CheckRefs checks a report against the registry: every Ref.ID declared by a +// case has an entry in the case's row (a deferred row keeps its entries), the +// row has no entry the registry does not declare, and every entry has a license. +func CheckRefs(rep *Report, cases []Case) []string { + rows := map[string]Row{} + for _, r := range rep.Rows { + rows[r.Name] = r + } + var bad []string + for _, c := range cases { + row, ok := rows[c.Name] + if !ok { + continue // CheckAllocs reports the missing row + } + have := map[string]RefRow{} + for _, r := range row.Refs { + have[r.ID] = r + } + want := map[string]bool{} + for _, r := range c.Refs { + want[r.ID] = true + e, ok := have[r.ID] + switch { + case !ok: + bad = append(bad, fmt.Sprintf("%s: reference %s has no entry in perf.json (record it)", c.Name, r.ID)) + case e.License != r.License: + bad = append(bad, fmt.Sprintf("%s: reference %s is licensed %q in perf.json, %q in the registry", c.Name, r.ID, e.License, r.License)) + } + } + for id := range have { + if !want[id] { + bad = append(bad, fmt.Sprintf("%s: perf.json has reference %s, which the case does not declare", c.Name, id)) + } + } + } + return bad +} + +// FirstDiff returns the index of the first value of a and b that differs in +// its bits, or -1. It is the check of the first multi-reference run: the +// primary reference, evaluated through the new path, must reproduce the +// fixtures' recorded column bit for bit, which proves the inputs reach the +// workers unaltered. +func FirstDiff(a, b []float64) int { + for i := range min(len(a), len(b)) { + if math.Float64bits(a[i]) != math.Float64bits(b[i]) { + return i + } + } + if len(a) != len(b) { + return min(len(a), len(b)) + } + return -1 +} + +// checkMultiRef validates the schema 2 fields of a row read from a file. +func checkMultiRef(row Row, schema int) error { + extra := len(row.Refs) > 0 || row.GoAccuracy != nil || row.PrimaryAccuracy != nil || row.GoBudget != 0 + if schema == SchemaVersion && extra { + return errors.New("a schema 1 file carries extra-reference fields (schema 2 is required)") + } + if !finiteNonNeg(row.GoBudget) { + return errors.New("go_budget is not a finite non-negative number") + } + for _, a := range []*AccResult{row.GoAccuracy, row.PrimaryAccuracy} { + if a != nil && !a.valid() { + return errors.New("an accuracy is out of range") + } + } + seen := map[string]bool{} + for _, r := range row.Refs { + switch { + case !refIDPattern.MatchString(r.ID): + return fmt.Errorf("reference ID %q is not :", r.ID) + case seen[r.ID]: + return fmt.Errorf("reference %q appears twice", r.ID) + case strings.TrimSpace(r.License) == "": + return fmt.Errorf("reference %q has no license", r.ID) + case r.Accuracy != nil && !r.Accuracy.valid(): + return fmt.Errorf("reference %q: an accuracy is out of range", r.ID) + } + seen[r.ID] = true + } + return nil +} + +// checkRoundRefs validates the extra samples of a round against its row. +func checkRoundRefs(rd Round, row Row) error { + if len(row.Refs) == 0 { + if len(rd.RefsNs) > 0 || len(rd.Order) > 0 { + return errors.New("a round has extra-reference samples but the row has no references") + } + return nil + } + if len(rd.RefsNs) != len(row.Refs) { + return errors.New("a round does not have one sample per reference") + } + want := map[string]bool{SideGo: true, SidePrimary: true} + for _, r := range row.Refs { + ns, ok := rd.RefsNs[r.ID] + if !ok || !finiteNonNeg(ns) || ns == 0 { + return fmt.Errorf("a round's sample of %s is not a finite positive time", r.ID) + } + want[r.ID] = true + } + if len(rd.Order) != len(want) { + return errors.New("a round's order is not a permutation of its sides") + } + got := map[string]bool{} + for _, s := range rd.Order { + if !want[s] || got[s] { + return fmt.Errorf("a round's order names %q twice or not at all", s) + } + got[s] = true + } + return nil +} + +// sameRefIDs reports whether two reference lists name the same IDs in the same order. +func sameRefIDs(a, b []RefRow) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i].ID != b[i].ID { + return false + } + } + return true +} + +// keepRefData returns old with the registry's identity (name, license) of cur. +// The lists have the same IDs (sameRefIDs). +func keepRefData(old, cur []RefRow) []RefRow { + out := append([]RefRow(nil), old...) + for i := range out { + out[i].Name, out[i].License = cur[i].Name, cur[i].License + } + return out +} diff --git a/multiref_test.go b/multiref_test.go new file mode 100644 index 0000000..0e1e252 --- /dev/null +++ b/multiref_test.go @@ -0,0 +1,544 @@ +package refbench + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "math" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +// fixedTimer returns a Timer of exactly ns nanoseconds per call. +func fixedTimer(ns int64, log *[]string, id string) Timer { + return func(iters int) (time.Duration, error) { + return time.Duration(ns * int64(iters)), nil + } +} + +func multiCase(log *[]string) Case { + return Case{ + Name: "gamma", Func: "Gamma", Reference: "scipy.special.gamma", Parity: ParityNone, + Go: spinFor(12000), + Ref: fixedTimer(10000, log, SidePrimary), + Refs: []Ref{ + {ID: "julia:Bessels.gamma", Name: "Bessels.gamma", License: "MIT", Timer: fixedTimer(5000, log, "julia:Bessels.gamma"), + Eval: func() ([]float64, error) { return []float64{1, 3}, nil }}, + {ID: "boost:tgamma", License: "BSL-1.0", Timer: fixedTimer(20000, log, "boost:tgamma")}, + }, + RefEval: func() ([]float64, error) { return []float64{1, 1}, nil }, + Accuracy: &Accuracy{ + Go: func() ([]float64, error) { return []float64{1, 1}, nil }, + Budget: 1, + Score: func(got []float64) AccResult { + return AccResult{MaxULP: ULP(got[1] - 1), MaxULPFin: ULP(got[1] - 1), Coverage: 1, Points: len(got)} + }, + }, + } +} + +func runMulti(t *testing.T, rounds int) (*Report, []string) { + t.Helper() + var log []string + c := multiCase(&log) + opt := Options{Rounds: rounds, MinTime: time.Millisecond, Load: func() (float64, error) { return 1, nil }, Poll: time.Millisecond} + rep, err := Run(context.Background(), []Case{c}, opt) + if err != nil { + t.Fatal(err) + } + return rep, log +} + +func TestRunRotatesSidesAndWritesSchema2(t *testing.T) { + rep, _ := runMulti(t, 4) + if rep.Schema != SchemaMultiRef { + t.Fatalf("schema = %d", rep.Schema) + } + row := rep.Rows[0] + if len(row.Rounds) != 4 { // 4 sides: 4 rounds are already a multiple + t.Fatalf("rounds = %d", len(row.Rounds)) + } + // Every side takes every position once per k rounds. + pos := map[string]map[int]int{} + for _, rd := range row.Rounds { + if len(rd.Order) != 4 { + t.Fatalf("order = %v", rd.Order) + } + for p, s := range rd.Order { + if pos[s] == nil { + pos[s] = map[int]int{} + } + pos[s][p]++ + } + } + for s, m := range pos { + for p := 0; p < 4; p++ { + if m[p] != 1 { + t.Errorf("side %s at position %d: %d times", s, p, m[p]) + } + } + } + // The best is the fastest side, the verdict against it. + if row.Best != "julia:Bessels.gamma" || row.BestVerdict != Slower { + t.Errorf("best = %q %q", row.Best, row.BestVerdict) + } +} + +func TestRoundsRoundedUpToTheNumberOfSides(t *testing.T) { + rep, _ := runMulti(t, 5) + if n := len(rep.Rows[0].Rounds); n != 8 { + t.Errorf("rounds = %d, want 8", n) + } +} + +func TestTwoSidesStillAlternate(t *testing.T) { + var log []string + c := multiCase(&log) + c.Refs, c.Accuracy, c.RefEval = nil, nil, nil + rep, err := Run(context.Background(), []Case{c}, Options{Rounds: 3, MinTime: time.Millisecond, Load: func() (float64, error) { return 1, nil }}) + if err != nil { + t.Fatal(err) + } + if rep.Schema != SchemaVersion || len(rep.Rows[0].Rounds) != 3 { + t.Fatalf("schema %d, %d rounds", rep.Schema, len(rep.Rows[0].Rounds)) + } + for i, rd := range rep.Rows[0].Rounds { + if rd.GoFirst != (i%2 == 0) || rd.Order != nil || rd.RefsNs != nil { + t.Errorf("round %d = %+v", i, rd) + } + } +} + +func TestPerReferenceVerdictsLabelsAndAccuracy(t *testing.T) { + rep, _ := runMulti(t, 4) + row := rep.Rows[0] + bessels, boost := row.Refs[0], row.Refs[1] + if bessels.Verdict != Slower || bessels.Ratio <= 1 { + t.Errorf("bessels = %+v", bessels) + } + // Bessels.gamma is faster than Go and less accurate than Go's stated bound + // (2 ulp against a budget of 1... Go has 0): a tradeoff, still a defect. + if bessels.Label != LabelTradeoff { + t.Errorf("label = %q", bessels.Label) + } + if boost.Verdict != Faster || boost.Label != "" { + t.Errorf("boost = %+v (no accuracy: no claim)", boost) + } + if row.GoAccuracy == nil || row.PrimaryAccuracy == nil || row.GoBudget != 1 { + t.Errorf("accuracy not recorded: %+v", row) + } + if bessels.Accuracy == nil || bessels.Accuracy.MaxULP != 2 { + t.Errorf("bessels accuracy = %+v", bessels.Accuracy) + } +} + +func TestLabel(t *testing.T) { + two := &AccResult{MaxULP: 2} + one := &AccResult{MaxULP: 1} + zero := &AccResult{} + cases := []struct { + v string + g, r *AccResult + budget float64 + want string + comment string + }{ + {Slower, one, one, 0, LabelBehind, "as accurate"}, + {Slower, one, two, 0, LabelTradeoff, "less accurate than Go's own max"}, + {Slower, two, one, 5, LabelBehind, "budget wider than the ref"}, + {Slower, one, two, 1.5, LabelTradeoff, "budget"}, + {Slower, nil, two, 0, LabelBehind, "no Go accuracy"}, + {Faster, one, two, 0, LabelAhead, ""}, + {Tie, one, one, 0, LabelAhead, ""}, + {Faster, two, one, 0, LabelFasterLessAccurate, ""}, + {Faster, zero, nil, 0, "", "no ref accuracy"}, + {"", one, one, 0, "", "no verdict"}, + } + for _, c := range cases { + if got := Label(c.v, c.g, c.r, c.budget); got != c.want { + t.Errorf("Label(%s, %v, %v, %v) = %q, want %q (%s)", c.v, c.g, c.r, c.budget, got, c.want, c.comment) + } + } +} + +func TestIssueAgainstAnyFasterReference(t *testing.T) { + rep, _ := runMulti(t, 4) + // Make the primary slower than Go (Go ties or beats it) and keep the + // less accurate Julia reference faster: the issue is still drafted. + issues, err := Issues(rep) + if err != nil || len(issues) != 1 { + t.Fatalf("issues = %v, %v", issues, err) + } + is := issues[0] + if !strings.Contains(is.Title, "Bessels.gamma") || !strings.Contains(is.Body, "tradeoff") || !strings.Contains(is.Body, "julia:Bessels.gamma") || !strings.Contains(is.Body, "BSL-1.0") { + t.Errorf("issue = %+v", is) + } + // Only a reference that is not the primary is faster. + var log []string + c := multiCase(&log) + c.Ref = fixedTimer(15000, &log, SidePrimary) // Go beats SciPy, but not Julia + rep2, err := Run(context.Background(), []Case{c}, Options{Rounds: 4, MinTime: time.Millisecond, Load: func() (float64, error) { return 1, nil }}) + if err != nil { + t.Fatal(err) + } + if rep2.Rows[0].Verdict != Faster { + t.Fatalf("setup: primary verdict %q", rep2.Rows[0].Verdict) + } + got, _ := Issues(rep2) + if len(got) != 1 || !strings.Contains(got[0].Title, "Bessels.gamma") { + t.Errorf("no issue for a row slower than Julia only: %v", got) + } +} + +func TestSchema2RoundTripAndTable(t *testing.T) { + rep, _ := runMulti(t, 4) + var b bytes.Buffer + if err := rep.Write(&b); err != nil { + t.Fatal(err) + } + js := b.String() + back, err := ReadReport(strings.NewReader(js)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(Table(back), Table(rep)) { + t.Error("table differs after a round trip") + } + tab := string(Table(back)) + for _, want := range []string{"## Against other references", "Bessels.gamma", "boost:tgamma", "tradeoff"} { + if !strings.Contains(tab, want) { + t.Errorf("table lacks %q:\n%s", want, tab) + } + } + // Derived fields are never trusted from the file. + s := strings.Replace(js, `"best": "julia:Bessels.gamma"`, `"best": "boost:tgamma"`, 1) + back, err = ReadReport(strings.NewReader(s)) + if err != nil || back.Rows[0].Best != "julia:Bessels.gamma" { + t.Errorf("best trusted from the file: %v", err) + } +} + +func TestTableDeferredAndInvalidRows(t *testing.T) { + var log []string + c := multiCase(&log) + rec, err := RecordAllocs([]Case{c}, Options{}) + if err != nil { + t.Fatal(err) + } + if rec.Schema != SchemaMultiRef || len(rec.Rows[0].Refs) != 2 { + t.Fatalf("deferred record: schema %d refs %v", rec.Schema, rec.Rows[0].Refs) + } + if tab := string(Table(rec)); !strings.Contains(tab, "deferred") { + t.Errorf("table = %s", tab) + } + rep, _ := runMulti(t, 4) + rep.Rows[0].Rounds[0].LoadAfter = 99 + if tab := string(Table(rep)); !strings.Contains(tab, "invalid") { + t.Errorf("table = %s", tab) + } + // A row without references in a schema 2 table is skipped by the section. + rep.Rows = append(rep.Rows, Row{Name: "plain", Func: "P"}) + if tab := string(Table(rep)); strings.Contains(tab, "| plain |") && strings.Contains(tab[strings.Index(tab, "## Against"):], "| plain |") { + t.Error("plain row in the references section") + } +} + +func TestULPJSON(t *testing.T) { + b, err := json.Marshal(AccResult{MaxULP: ULP(math.Inf(1)), MaxULPFin: 2.5, Coverage: 0.5, Points: 4}) + if err != nil || !strings.Contains(string(b), `"max_ulp":"inf"`) || !strings.Contains(string(b), `"max_ulp_fin":2.5`) { + t.Fatalf("%s %v", b, err) + } + var a AccResult + if err := json.Unmarshal(b, &a); err != nil || !math.IsInf(float64(a.MaxULP), 1) || a.MaxULPFin != 2.5 { + t.Fatalf("%+v %v", a, err) + } + for _, bad := range []ULP{ULP(math.NaN()), ULP(math.Inf(-1)), -1} { + if _, err := json.Marshal(bad); err == nil { + t.Errorf("%v encoded", bad) + } + } + if err := json.Unmarshal([]byte(`"nan"`), new(ULP)); err == nil { + t.Error(`"nan" decoded`) + } +} + +func TestValidateRefs(t *testing.T) { + base := func() Case { + return Case{Name: "a", Func: "A", Parity: ParityNone, Go: func(int) {}, Refs: []Ref{{ID: "julia:x", License: "MIT"}}} + } + if err := Validate([]Case{base()}); err != nil { + t.Fatal(err) + } + mut := map[string]func(*Case){ + "id": func(c *Case) { c.Refs[0].ID = "nokind" }, + "dup": func(c *Case) { c.Refs = append(c.Refs, c.Refs[0]) }, + "license": func(c *Case) { c.Refs[0].License = " " }, + "gpl": func(c *Case) { c.Refs[0].License = "GPL-3.0" }, + "lgpl": func(c *Case) { c.Refs[0].License = "LGPL-3.0" }, + "acc": func(c *Case) { c.Accuracy = &Accuracy{} }, + "budget": func(c *Case) { + c.Accuracy = &Accuracy{Go: func() ([]float64, error) { return nil, nil }, Score: func([]float64) AccResult { return AccResult{} }, Budget: -1} + }, + "reserved": func(c *Case) { c.Refs[0].ID = "go:primary" }, // valid: kind "go" is just a kind + } + for name, f := range mut { + c := base() + f(&c) + err := Validate([]Case{c}) + if name == "reserved" { + if err != nil { + t.Errorf("%s: %v", name, err) + } + continue + } + if err == nil { + t.Errorf("%s: accepted", name) + } + } + // A GPL reference is fine when it is a black-box kind. + c := base() + c.Refs[0] = Ref{ID: "r:besselI", License: "GPL-black-box"} + if err := Validate([]Case{c}); err != nil { + t.Errorf("black box refused: %v", err) + } + if RefKind("julia:x") != "julia" { + t.Error("RefKind") + } +} + +func TestRunRefErrors(t *testing.T) { + var log []string + c := multiCase(&log) + c.Refs[0].Timer = nil + if _, err := Run(context.Background(), []Case{c}, Options{}); err == nil || !strings.Contains(err.Error(), "no timer") { + t.Errorf("nil ref timer: %v", err) + } + boom := errors.New("boom") + for name, mut := range map[string]func(*Case){ + "ref timer": func(c *Case) { c.Refs[1].Timer = func(int) (time.Duration, error) { return 0, boom } }, + "go eval": func(c *Case) { c.Accuracy.Go = func() ([]float64, error) { return nil, boom } }, + "ref eval": func(c *Case) { c.Refs[0].Eval = func() ([]float64, error) { return nil, boom } }, + "primary": func(c *Case) { c.RefEval = func() ([]float64, error) { return nil, boom } }, + "scorer": func(c *Case) { + c.Accuracy.Score = func([]float64) AccResult { return AccResult{Coverage: 2} } + }, + } { + var l []string + c := multiCase(&l) + mut(&c) + _, err := Run(context.Background(), []Case{c}, Options{Rounds: 2, MinTime: time.Millisecond, Load: func() (float64, error) { return 1, nil }}) + if err == nil { + t.Errorf("%s: no error", name) + } + } +} + +func TestRetryAndGateDiscardAllSides(t *testing.T) { + var log []string + c := multiCase(&log) + var n atomic.Int32 + load := func() (float64, error) { + if n.Add(1) == 4 { // the first round ends above the gate once + return 99, nil + } + return 1, nil + } + rep, err := Run(context.Background(), []Case{c}, Options{Rounds: 4, MinTime: time.Millisecond, Load: load, OnLoad: OnLoadWait, Poll: time.Millisecond, LoadTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + if rep.Rows[0].Retries != 1 || len(rep.Rows[0].Rounds) != 4 { + t.Errorf("retries %d rounds %d", rep.Rows[0].Retries, len(rep.Rows[0].Rounds)) + } + load = func() (float64, error) { return 1, errors.New("load") } + if _, err := Run(context.Background(), []Case{c}, Options{Rounds: 2, MinTime: time.Millisecond, Load: load}); err == nil { + t.Error("load error lost") + } + // The retry cap defers the row. + load = func() (float64, error) { return 1, nil } + always := func() (float64, error) { return 99, nil } + _ = load + if _, err := Run(context.Background(), []Case{c}, Options{Rounds: 2, MinTime: time.Millisecond, Load: always, OnLoad: OnLoadRefuse}); !errors.Is(err, ErrOnLoad) { + t.Errorf("err = %v", err) + } +} + +func TestReadReportRejectsBadSchema2(t *testing.T) { + rep, _ := runMulti(t, 4) + var b bytes.Buffer + if err := rep.Write(&b); err != nil { + t.Fatal(err) + } + good := b.String() + mut := func(old, new string) string { + if !strings.Contains(good, old) { + t.Fatalf("%q not in the file", old) + } + return strings.Replace(good, old, new, 1) + } + for name, in := range map[string]string{ + "schema1": mut(`"schema": 2`, `"schema": 1`), + "badid": mut(`"id": "boost:tgamma"`, `"id": "boost"`), + "dupid": mut(`"id": "boost:tgamma"`, `"id": "julia:Bessels.gamma"`), + "nolicense": mut(`"license": "BSL-1.0"`, `"license": ""`), + "coverage": mut(`"coverage": 1`, `"coverage": 3`), + "budget": mut(`"go_budget": 1`, `"go_budget": -1`), + "refs_ns": mut(`"julia:Bessels.gamma": 5000`, `"julia:Bessels.gamma": 0`), + "order": mut(`"order": [`, `"order": ["go", `), + "missing ns": mut(`"boost:tgamma": 20000`, `"x:y": 20`), + } { + if _, err := ReadReport(strings.NewReader(in)); err == nil { + t.Errorf("%s: accepted", name) + } + } + // A round with references on a row with none. + plain := `{"schema":2,"status":"measured","max_load":8,"rows":[{"name":"a","func":"F","allocs":0,"rounds":[{"load_before":1,"load_after":1,"go_ns":1,"ref_ns":1,"refs_ns":{"x:y":1}}]}]}` + if _, err := ReadReport(strings.NewReader(plain)); err == nil { + t.Error("orphan refs_ns accepted") + } + // Order naming a side twice. + dup := strings.Replace(good, `"order": [`, `"order": ["go", "go", "primary", "x:y"], "x": [`, 1) + if _, err := ReadReport(strings.NewReader(dup)); err == nil { + t.Error("bad order accepted") + } + if bad := (AccResult{Coverage: math.NaN()}); bad.valid() { + t.Error("NaN coverage valid") + } +} + +func TestCheckRefsAndRunCI(t *testing.T) { + var log []string + c := multiCase(&log) + rec, err := RecordAllocs([]Case{c}, Options{}) + if err != nil { + t.Fatal(err) + } + if bad := CheckRefs(rec, []Case{c}); len(bad) != 0 { + t.Errorf("clean report: %v", bad) + } + c2 := c + c2.Refs = append(append([]Ref(nil), c.Refs...), Ref{ID: "fortran:amos", License: "LicenseRef-public-domain"}) + c2.Refs[0].License = "Apache-2.0" + c3 := c + c3.Refs = c.Refs[:1] + for _, bad := range [][]string{CheckRefs(rec, []Case{c2}), CheckRefs(rec, []Case{c3})} { + if len(bad) == 0 { + t.Error("a registry that differs from the report passed") + } + } + if bad := CheckRefs(rec, []Case{{Name: "absent"}}); len(bad) != 0 { + t.Errorf("absent row: %v", bad) + } + + // RunCI end to end: no worker, no timing. + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "docs"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module x\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "x.go"), []byte("package x\n\nfunc Gamma() {}\n"), 0o644); err != nil { + t.Fatal(err) + } + var js bytes.Buffer + if err := rec.Write(&js); err != nil { + t.Fatal(err) + } + cfg := CIConfig{Root: dir, Cases: []Case{c}} + jp, tp := cfg.Paths() + if err := os.WriteFile(jp, js.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(tp, Table(rec), 0o644); err != nil { + t.Fatal(err) + } + if err := RunCI(cfg); err != nil { + t.Fatalf("CI: %v", err) + } + cfg.Cases = []Case{c2} + if err := RunCI(cfg); err == nil || !strings.Contains(err.Error(), "references:") { + t.Errorf("CI with a changed registry: %v", err) + } +} + +func TestMergeAllocsKeepsOrDefersByReferenceSet(t *testing.T) { + rep, _ := runMulti(t, 4) + var log []string + c := multiCase(&log) + rec, _ := RecordAllocs([]Case{c}, Options{}) + same := MergeAllocs(rep, rec) + if len(same.Rows[0].Rounds) != 4 || same.Schema != SchemaMultiRef { + t.Errorf("same set: %d rounds, schema %d", len(same.Rows[0].Rounds), same.Schema) + } + c.Refs = c.Refs[:1] + c.Refs[0].License = "Apache-2.0" + rec, _ = RecordAllocs([]Case{c}, Options{}) + kept := MergeAllocs(rep, rec) + if len(kept.Rows[0].Rounds) != 0 || kept.Valid { + t.Errorf("changed set kept its rounds: %+v", kept.Rows[0]) + } + c.Refs = nil + rec, _ = RecordAllocs([]Case{c}, Options{}) + if gone := MergeAllocs(rep, rec); gone.Schema != SchemaVersion { + t.Errorf("schema = %d after dropping every reference", gone.Schema) + } + // Licence of a kept set follows the registry. + c = multiCase(&log) + c.Refs[1].License = "MIT" + rec, _ = RecordAllocs([]Case{c}, Options{}) + if got := MergeAllocs(rep, rec); got.Rows[0].Refs[1].License != "MIT" || len(got.Rows[0].Rounds) != 4 { + t.Errorf("licence not updated: %+v", got.Rows[0].Refs) + } +} + +func TestFirstDiff(t *testing.T) { + nan := math.NaN() + for _, c := range []struct { + a, b []float64 + want int + }{ + {[]float64{1, 2}, []float64{1, 2}, -1}, + {[]float64{1, 2}, []float64{1, 3}, 1}, + {[]float64{0}, []float64{math.Copysign(0, -1)}, 0}, + {[]float64{nan}, []float64{nan}, -1}, + {[]float64{1}, []float64{1, 2}, 1}, + {nil, nil, -1}, + } { + if got := FirstDiff(c.a, c.b); got != c.want { + t.Errorf("FirstDiff(%v, %v) = %d, want %d", c.a, c.b, got, c.want) + } + } +} + +func TestCalibrateDiscardsTheFirstCall(t *testing.T) { + // A JIT-like timer: its first call lasts 1 s, the others take 1 us per call. + calls := 0 + tm := func(iters int) (time.Duration, error) { + calls++ + if calls == 1 { + return time.Second, nil + } + return time.Duration(iters) * time.Microsecond, nil + } + got, err := Calibrate(tm, 10*time.Millisecond) + if err != nil || got < 10000 { + t.Errorf("iters = %d, %v: the first call was used", got, err) + } + if _, err := Calibrate(func(int) (time.Duration, error) { return 0, errors.New("x") }, time.Millisecond); err == nil { + t.Error("warm-up error lost") + } +} + +// spinFor returns a Go body that lasts ns nanoseconds per iteration, by busy waiting. +func spinFor(ns int64) func(int) { + return func(iters int) { + for end := time.Now().Add(time.Duration(ns * int64(iters))); time.Now().Before(end); { + } + } +} diff --git a/rbtest/rbtest_test.go b/rbtest/rbtest_test.go index c788b9b..97a8a50 100644 --- a/rbtest/rbtest_test.go +++ b/rbtest/rbtest_test.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -334,7 +335,7 @@ func TestPartialThenResume(t *testing.T) { t.Fatalf("resumed = %+v", done) } for i, rd := range done.Rows[0].Rounds { - if rd != rep.Rows[0].Rounds[i] { + if !reflect.DeepEqual(rd, rep.Rows[0].Rounds[i]) { t.Errorf("row a round %d changed: %+v -> %+v", i, rep.Rows[0].Rounds[i], rd) } } diff --git a/report.go b/report.go index 31c65fb..b23d9c9 100644 --- a/report.go +++ b/report.go @@ -30,6 +30,11 @@ type Round struct { // GoNs and RefNs are nanoseconds per call. GoNs float64 `json:"go_ns"` RefNs float64 `json:"ref_ns"` + // RefsNs is the nanoseconds per call of every extra reference, by ID, and + // Order the sides in the order they ran this round (SideGo, SidePrimary and + // the IDs). Schema 2 only. + RefsNs map[string]float64 `json:"refs_ns,omitempty"` + Order []string `json:"order,omitempty"` } // Row is one case's result. @@ -45,6 +50,22 @@ type Row struct { // Retries counts the rounds of this row that ended above the load gate and // were discarded and re-run (across every run that measured the row). Retries int `json:"retries,omitempty"` + // Refs are the extra references (schema 2): identity from the registry, + // the rest derived. GoAccuracy and PrimaryAccuracy are the accuracy of Go + // and of the primary reference on the case's points, GoBudget Go's stated + // bound in ulp. They are measured once, ungated, before the rounds. + Refs []RefRow `json:"refs,omitempty"` + GoAccuracy *AccResult `json:"go_accuracy,omitempty"` + PrimaryAccuracy *AccResult `json:"primary_accuracy,omitempty"` + GoBudget float64 `json:"go_budget,omitempty"` + + // Best is the fastest side by median (SidePrimary or a reference ID), and + // BestRatio, BestVerdict and BestLabel Go against it: the row's headline + // (derived; empty without Refs). + Best string `json:"best,omitempty"` + BestRatio float64 `json:"best_ratio,omitempty"` + BestVerdict string `json:"best_verdict,omitempty"` + BestLabel string `json:"best_label,omitempty"` // Derived from Rounds by Finish; never trusted from a file. // Valid: the row has rounds and every one started and ended under the gate. @@ -138,6 +159,18 @@ func (r *Report) Finish() { row.Ratio = row.Go.Median / row.Ref.Median row.Verdict = Judge(row.Go, row.Ref) } + for i := range r.Rows { + // After the loop: finishRefs reads the row's Go and primary summaries, + // also when the row has no rounds (it then clears the derived fields). + finishRefs(&r.Rows[i]) + } + if r.Schema == SchemaVersion || r.Schema == SchemaMultiRef { + // The schema is what the content needs: 2 with extra references, else 1. + r.Schema = SchemaVersion + if r.usesRefs() { + r.Schema = SchemaMultiRef + } + } if r.Status == StatusMeasured { r.Valid = len(r.Deferred) == 0 } else { @@ -184,8 +217,8 @@ func ReadReport(rd io.Reader) (*Report, error) { if dec.More() { return nil, errors.New("perf.json: trailing data after the object") } - if r.Schema != SchemaVersion { - return nil, fmt.Errorf("perf.json: schema %d, this build reads %d", r.Schema, SchemaVersion) + if r.Schema != SchemaVersion && r.Schema != SchemaMultiRef { + return nil, fmt.Errorf("perf.json: schema %d, this build reads %d and %d", r.Schema, SchemaVersion, SchemaMultiRef) } if r.Status != StatusMeasured && r.Status != StatusDeferred { return nil, fmt.Errorf("perf.json: unknown status %q", r.Status) @@ -195,7 +228,7 @@ func ReadReport(rd io.Reader) (*Report, error) { } seen := map[string]bool{} for i, row := range r.Rows { - if err := checkRow(row, r.Status); err != nil { + if err := checkRow(row, r.Status, r.Schema); err != nil { return nil, fmt.Errorf("perf.json: row %d: %w", i, err) } if seen[row.Name] { @@ -209,7 +242,10 @@ func ReadReport(rd io.Reader) (*Report, error) { func finiteNonNeg(x float64) bool { return !math.IsNaN(x) && !math.IsInf(x, 0) && x >= 0 } -func checkRow(row Row, status string) error { +func checkRow(row Row, status string, schema int) error { + if err := checkMultiRef(row, schema); err != nil { + return err + } switch { case row.Name == "" || row.Func == "": return errors.New("name and func are required") @@ -221,6 +257,9 @@ func checkRow(row Row, status string) error { return errors.New("retries is negative") } for _, rd := range row.Rounds { + if err := checkRoundRefs(rd, row); err != nil { + return err + } if !finiteNonNeg(rd.LoadBefore) || !finiteNonNeg(rd.LoadAfter) { return errors.New("a round's load is not a finite non-negative number") } diff --git a/report_test.go b/report_test.go index 82707a8..cac7050 100644 --- a/report_test.go +++ b/report_test.go @@ -86,22 +86,23 @@ func TestReadReportRejects(t *testing.T) { good := ok.String() mut := func(old, new string) string { return strings.Replace(good, old, new, 1) } for name, in := range map[string]string{ - "empty": ``, - "unknown": mut(`"schema": 1`, `"schema": 1, "extra": 1`), - "trailing": good + `{}`, - "schema": mut(`"schema": 1`, `"schema": 2`), - "status": mut(`"status": "measured"`, `"status": "wat"`), - "maxload": mut(`"max_load": 8`, `"max_load": -1`), - "noname": mut(`"name": "fast"`, `"name": ""`), - "dup": mut(`"name": "fast"`, `"name": "slow|row"`), - "allocs": mut(`"allocs": 2`, `"allocs": -2`), - "load": mut(`"load_before": 3`, `"load_before": -3`), - "sample": mut(`"go_ns": 110`, `"go_ns": -110`), - "zeroref": mut(`"ref_ns": 11`, `"ref_ns": 0`), - "retries": mut(`"allocs": 2`, `"allocs": 2, "retries": -1`), - "deferredR": mut(`"status": "measured"`, `"status": "deferred"`), - "loadafter": mut(`"load_after": 4`, `"load_after": -4`), - "loadbefore": mut(`"load_before": 1,`, `"load_before": -1,`), + "empty": ``, + "unknown": mut(`"schema": 1`, `"schema": 1, "extra": 1`), + "trailing": good + `{}`, + "schema": mut(`"schema": 1`, `"schema": 3`), + "schema1refs": mut(`"allocs": 2`, `"allocs": 2, "go_budget": 1`), + "status": mut(`"status": "measured"`, `"status": "wat"`), + "maxload": mut(`"max_load": 8`, `"max_load": -1`), + "noname": mut(`"name": "fast"`, `"name": ""`), + "dup": mut(`"name": "fast"`, `"name": "slow|row"`), + "allocs": mut(`"allocs": 2`, `"allocs": -2`), + "load": mut(`"load_before": 3`, `"load_before": -3`), + "sample": mut(`"go_ns": 110`, `"go_ns": -110`), + "zeroref": mut(`"ref_ns": 11`, `"ref_ns": 0`), + "retries": mut(`"allocs": 2`, `"allocs": 2, "retries": -1`), + "deferredR": mut(`"status": "measured"`, `"status": "deferred"`), + "loadafter": mut(`"load_after": 4`, `"load_after": -4`), + "loadbefore": mut(`"load_before": 1,`, `"load_before": -1,`), } { if _, err := ReadReport(strings.NewReader(in)); err == nil { t.Errorf("%s: accepted", name) diff --git a/resume.go b/resume.go index 32777b2..91204c0 100644 --- a/resume.go +++ b/resume.go @@ -106,10 +106,18 @@ func MergeAllocs(prev, rec *Report) *Report { } out := *prev out.Commit = rec.Commit // the allocation counts are those of this commit + out.Schema = rec.Schema // Finish raises it to 2 when a row keeps extra references out.Rows = make([]Row, 0, len(rec.Rows)) for _, r := range rec.Rows { if o, ok := old[r.Name]; ok { o.Func, o.Reference, o.Parity, o.Allocs = r.Func, r.Reference, r.Parity, r.Allocs + if sameRefIDs(o.Refs, r.Refs) { + o.Refs = keepRefData(o.Refs, r.Refs) + } else { + // The set of references changed: the recorded rounds measured + // another set and cannot be kept. The row is deferred again. + o.Rounds, o.Refs, o.GoAccuracy, o.PrimaryAccuracy, o.GoBudget = nil, r.Refs, nil, nil, 0 + } r = o } out.Rows = append(out.Rows, r) diff --git a/runner.go b/runner.go index 394b3f7..238834b 100644 --- a/runner.go +++ b/runner.go @@ -127,7 +127,7 @@ func AllocsPerCall(c Case) float64 { } func (c Case) row(allocs float64) Row { - return Row{Name: c.Name, Func: c.Func, Reference: c.Reference, Parity: c.Parity, Allocs: allocs} + return Row{Name: c.Name, Func: c.Func, Reference: c.Reference, Parity: c.Parity, Allocs: allocs, Refs: c.refRows()} } // RecordAllocs returns a deferred report: the allocation counts of every case @@ -156,7 +156,9 @@ func goTimer(c Case) Timer { // Run measures every case. The Go side runs under GOMAXPROCS=1, set before and // restored after, never inside a timed interval. Rounds alternate the order of -// the two sides (Go first on even rounds, reference first on odd ones); the load +// the two sides (Go first on even rounds, reference first on odd ones); with +// extra references (Case.Refs) the sides are rotated instead, and Rounds is +// rounded up to a multiple of the number of sides; the load // is read, gated and stored before every round and read again after it. A // reference error stops the run: a series with a hole is biased, not smaller. // @@ -176,6 +178,11 @@ func Run(ctx context.Context, cases []Case, opt Options) (*Report, error) { if c.Ref == nil { return nil, fmt.Errorf("case %q has no reference timer", c.Name) } + for _, r := range c.Refs { + if r.Timer == nil { + return nil, fmt.Errorf("case %q: reference %s has no timer", c.Name, r.ID) + } + } } opt = opt.withDefaults() defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) @@ -215,6 +222,9 @@ var errRetryCap = errors.New("rounds kept ending above the load gate") func runCase(ctx context.Context, c Case, opt Options) (Row, error) { row := c.row(AllocsPerCall(c)) + if err := scoreAccuracy(c, &row); err != nil { + return row, err + } g, f := goTimer(c), c.Ref var err error if row.GoIters, err = Calibrate(g, opt.MinTime); err != nil { @@ -223,33 +233,61 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { if row.RefIters, err = Calibrate(f, opt.MinTime); err != nil { return row, err } - for r := 0; r < opt.Rounds; { + // The sides, in the order of the registry: Go, the primary, the extra ones. + // Round r runs them rotated by r mod len(sides), so that every side takes + // every position once per len(sides) rounds. With two sides this is the + // alternation of schema 1 (Go first on even rounds). + sides := []roundSide{ + {id: SideGo, t: g, iters: row.GoIters}, + {id: SidePrimary, t: f, iters: row.RefIters}, + } + for i, r := range c.Refs { + it, err := Calibrate(r.Timer, opt.MinTime) + if err != nil { + return row, fmt.Errorf("%s: %w", r.ID, err) + } + row.Refs[i].Iters = it + sides = append(sides, roundSide{id: r.ID, t: r.Timer, iters: it}) + } + k := len(sides) + rounds := opt.Rounds + if k > 2 && rounds%k != 0 { + rounds += k - rounds%k + } + for r := 0; r < rounds; { load, err := opt.gate(ctx) if err != nil { return row, err } - rd := Round{LoadBefore: load, GoFirst: r%2 == 0} - sides := [2]struct { - t Timer - iters int - out *float64 - }{{g, row.GoIters, &rd.GoNs}, {f, row.RefIters, &rd.RefNs}} - if !rd.GoFirst { - sides[0], sides[1] = sides[1], sides[0] + rd := Round{LoadBefore: load, GoFirst: sideIndex(sides, r%k, SideGo) < sideIndex(sides, r%k, SidePrimary)} + if k > 2 { + rd.RefsNs = make(map[string]float64, k-2) } - for _, s := range sides { + for p := 0; p < k; p++ { + s := sides[(r+p)%k] d, err := s.t(s.iters) if err != nil { return row, err } - *s.out = float64(d.Nanoseconds()) / float64(s.iters) + ns := float64(d.Nanoseconds()) / float64(s.iters) + switch s.id { + case SideGo: + rd.GoNs = ns + case SidePrimary: + rd.RefNs = ns + default: + rd.RefsNs[s.id] = ns + } + if k > 2 { + rd.Order = append(rd.Order, s.id) + } } if rd.LoadAfter, err = opt.Load(); err != nil { return row, fmt.Errorf("cannot read the load average: %w", err) } if rd.LoadAfter >= opt.MaxLoad { - // Discarded; the same round (same side order) is run again once the - // gate is open. + // Discarded for every side; the same round (same side order) is run + // again once the gate is open. row.Retries++ if row.Retries > opt.MaxRetries { return row, errRetryCap @@ -261,3 +299,59 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { } return row, nil } + +type roundSide struct { + id string + t Timer + iters int +} + +// sideIndex returns the position of side id in the round whose rotation is rot. +func sideIndex(sides []roundSide, rot int, id string) int { + k := len(sides) + for p := 0; p < k; p++ { + if sides[(rot+p)%k].id == id { + return p + } + } + return -1 +} + +// scoreAccuracy evaluates Go, the primary reference and every Ref with an Eval +// on the case's points and stores the results in the row. It is deterministic, +// so it is not gated; an evaluation error stops the run (a series with a hole +// is biased). +func scoreAccuracy(c Case, row *Row) error { + a := c.Accuracy + if a == nil { + return nil + } + score := func(who string, eval EvalFunc) (*AccResult, error) { + if eval == nil { + return nil, nil + } + got, err := eval() + if err != nil { + return nil, fmt.Errorf("accuracy of %s: %w", who, err) + } + res := a.Score(got) + if !res.valid() { + return nil, fmt.Errorf("accuracy of %s: the scorer returned an out-of-range result", who) + } + return &res, nil + } + var err error + if row.GoAccuracy, err = score("Go", a.Go); err != nil { + return err + } + if row.PrimaryAccuracy, err = score(SidePrimary, c.RefEval); err != nil { + return err + } + row.GoBudget = a.Budget + for i, r := range c.Refs { + if row.Refs[i].Accuracy, err = score(r.ID, r.Eval); err != nil { + return err + } + } + return nil +} diff --git a/table.go b/table.go index 9fdcff5..abac8a5 100644 --- a/table.go +++ b/table.go @@ -54,9 +54,108 @@ func Table(r *Report) []byte { fmt.Fprintf(&b, "| %s | `%s` | %s | %s | %s | %s | %s | %s | %g |\n", esc(row.Name), row.Func, esc(row.Reference), goCell, refCell, ratio, verdict, row.Parity, row.Allocs) } + if r.usesRefs() { + refsSection(&b, r) + } return b.Bytes() } +// refsSection is the second section of a schema 2 table: Go against every other +// reference, with the accuracy of each side. The first section is unchanged. +func refsSection(b *bytes.Buffer, r *Report) { + var ids []string + names := map[string]string{} + for _, row := range r.Rows { + for _, ref := range row.Refs { + if _, ok := names[ref.ID]; !ok { + ids = append(ids, ref.ID) + names[ref.ID] = ref.label() + } + } + } + sort.Strings(ids) + b.WriteString("\n## Against other references\n\n") + b.WriteString("Ratio = Go / reference, from the same rounds as the table above. Each cell: ratio (verdict, label; max ulp, coverage). The headline is Go against the fastest side, primary reference included: a reference that is faster is a defect even when it is less accurate.\n\n") + b.WriteString("| case | Go ns | Go max ulp | best | ratio vs best | label |") + for _, id := range ids { + fmt.Fprintf(b, " %s |", esc(names[id])) + } + b.WriteString("\n|---|---:|---:|---|---:|---|") + for range ids { + b.WriteString("---|") + } + b.WriteString("\n") + for _, row := range r.Rows { + if len(row.Refs) == 0 { + continue + } + goNs, goUlp, best, ratio, label := "—", ulpCell(row.GoAccuracy), "—", "—", "deferred" + if len(row.Rounds) > 0 { + goNs, best, ratio, label = fmt.Sprintf("%.4g", row.Go.Median), esc(sideName(row, row.Best)), fmt.Sprintf("%.2f", row.BestRatio), row.BestVerdict + if row.BestLabel != "" { + label += ", " + row.BestLabel + } + if !row.Valid { + label = "invalid" + } + } + fmt.Fprintf(b, "| %s | %s | %s | %s | %s | %s |", esc(row.Name), goNs, goUlp, best, ratio, label) + byID := map[string]RefRow{} + for _, ref := range row.Refs { + byID[ref.ID] = ref + } + for _, id := range ids { + ref, ok := byID[id] + switch { + case !ok: + b.WriteString(" — |") + case len(row.Rounds) == 0: + fmt.Fprintf(b, " deferred (%s) |", accCell(ref.Accuracy)) + default: + v := ref.Verdict + if ref.Label != "" { + v += ", " + ref.Label + } + fmt.Fprintf(b, " %.2f (%s; %s) |", ref.Ratio, v, accCell(ref.Accuracy)) + } + } + b.WriteString("\n") + } +} + +func sideName(row Row, id string) string { + if id == SidePrimary { + return row.Reference + } + for _, ref := range row.Refs { + if ref.ID == id { + return ref.label() + } + } + return id +} + +func ulpCell(a *AccResult) string { + if a == nil { + return "—" + } + return fmt.Sprintf("%.3g", float64(a.MaxULP)) +} + +func covCell(a *AccResult) string { + if a == nil { + return "—" + } + return fmt.Sprintf("%.0f%%", 100*a.Coverage) +} + +func accCell(a *AccResult) string { + if a == nil { + return "no accuracy" + } + return fmt.Sprintf("%.3g ulp, cov %.0f%%", float64(a.MaxULP), 100*a.Coverage) +} + func cell(s Summary) string { return fmt.Sprintf("%.4g [%.4g, %.4g]", s.Median, s.Q1, s.Q3) } diff --git a/testdata/golden/schema1-table.md b/testdata/golden/schema1-table.md new file mode 100644 index 0000000..ac33ac7 --- /dev/null +++ b/testdata/golden/schema1-table.md @@ -0,0 +1,19 @@ +# Performance table + + + +- Date: 2026-11-07T12:00:00Z +- Commit: abc +- Go: go1.27.3 +- Machine: m +- blas: accelerate +- numpy: 2.5.4 +- Load (1 min) over the rounds: min 1.00, median 1.50, max 4.00; gate 8.00 +- Run: valid (every round under the gate) + +ns/call, median [first quartile, third quartile]; ratio = Go / reference. + +| case | function | reference | Go | reference | ratio | verdict | parity | allocs | +|---|---|---|---:|---:|---:|---|---|---:| +| slow\|row | `F` | ref\|x | 105 [102.5, 107.5] | 10.5 [10.25, 10.75] | 10.00 | slower | exact | 2 | +| fast | `G` | ref | 1 [1, 1] | 10 [10, 10] | 0.10 | faster | tolerance | 0 | diff --git a/testdata/golden/schema1.json b/testdata/golden/schema1.json new file mode 100644 index 0000000..1801a57 --- /dev/null +++ b/testdata/golden/schema1.json @@ -0,0 +1,89 @@ +{ + "schema": 1, + "status": "measured", + "date": "2026-11-07T12:00:00Z", + "commit": "abc", + "repo": "go-x/y", + "go_version": "go1.27.3", + "machine": "m", + "ref_versions": { + "blas": "accelerate", + "numpy": "2.5.4" + }, + "max_load": 8, + "load": { + "min": 1, + "median": 1.5, + "max": 4 + }, + "valid": true, + "rows": [ + { + "name": "slow|row", + "func": "F", + "reference": "ref|x", + "parity": "exact", + "allocs": 2, + "go_iters": 10, + "ref_iters": 10, + "rounds": [ + { + "load_before": 1, + "load_after": 2, + "go_first": true, + "go_ns": 100, + "ref_ns": 10 + }, + { + "load_before": 3, + "load_after": 4, + "go_first": false, + "go_ns": 110, + "ref_ns": 11 + } + ], + "valid": true, + "go": { + "median": 105, + "q1": 102.5, + "q3": 107.5 + }, + "ref": { + "median": 10.5, + "q1": 10.25, + "q3": 10.75 + }, + "ratio": 10, + "verdict": "slower" + }, + { + "name": "fast", + "func": "G", + "reference": "ref", + "parity": "tolerance", + "allocs": 0, + "rounds": [ + { + "load_before": 1, + "load_after": 1, + "go_first": true, + "go_ns": 1, + "ref_ns": 10 + } + ], + "valid": true, + "go": { + "median": 1, + "q1": 1, + "q3": 1 + }, + "ref": { + "median": 10, + "q1": 10, + "q3": 10 + }, + "ratio": 0.1, + "verdict": "faster" + } + ] +} From e66371b809984e6106540a34678ade237666ad0a Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 15:17:25 +0100 Subject: [PATCH 02/12] rbw1 worker protocol, Julia worker, compiled-worker skeleton The rbw1 plain-text protocol (L, W, T, E, V), strict reply parser with a fuzz target and committed corpus, 4 KiB line bound (refused, tested under a memory limit), fixed function table, paths inside the run directory, float64 argument files verified by SHA-256 before any timing, single-thread checked against what the worker reports. A Julia worker (julialang.org binary through pkgx, pinned; version and Manifest tree hashes recorded under julia/ keys) and a C++/Fortran harness with examples; BuildWorker records source digests and flags. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- multiref_more_test.go | 131 ++++ rbw1.go | 518 +++++++++++++ rbw1_test.go | 733 ++++++++++++++++++ runner.go | 15 +- table.go | 2 +- .../FuzzParseRBW1Reply/seed-doubled-space | 2 + .../fuzz/FuzzParseRBW1Reply/seed-err-nonascii | 2 + testdata/fuzz/FuzzParseRBW1Reply/seed-load-ok | 2 + .../fuzz/FuzzParseRBW1Reply/seed-ns-overflow | 2 + .../fuzz/FuzzParseRBW1Reply/seed-versions | 2 + testdata/fuzz/FuzzReadReport/seed-schema2 | 2 + .../fuzz/FuzzReadReport/seed-schema2-bad-ulp | 2 + workers.go | 118 +++ workers/compiled/examples/cpp/kernels.cpp | 22 + workers/compiled/examples/fortran/sqrt.f90 | 15 + workers/compiled/examples/fortran/table.cpp | 15 + workers/compiled/rbw1.cpp | 265 +++++++ workers/julia/worker.jl | 146 ++++ 18 files changed, 1981 insertions(+), 13 deletions(-) create mode 100644 multiref_more_test.go create mode 100644 rbw1.go create mode 100644 rbw1_test.go create mode 100644 testdata/fuzz/FuzzParseRBW1Reply/seed-doubled-space create mode 100644 testdata/fuzz/FuzzParseRBW1Reply/seed-err-nonascii create mode 100644 testdata/fuzz/FuzzParseRBW1Reply/seed-load-ok create mode 100644 testdata/fuzz/FuzzParseRBW1Reply/seed-ns-overflow create mode 100644 testdata/fuzz/FuzzParseRBW1Reply/seed-versions create mode 100644 testdata/fuzz/FuzzReadReport/seed-schema2 create mode 100644 testdata/fuzz/FuzzReadReport/seed-schema2-bad-ulp create mode 100644 workers.go create mode 100644 workers/compiled/examples/cpp/kernels.cpp create mode 100644 workers/compiled/examples/fortran/sqrt.f90 create mode 100644 workers/compiled/examples/fortran/table.cpp create mode 100644 workers/compiled/rbw1.cpp create mode 100644 workers/julia/worker.jl diff --git a/multiref_more_test.go b/multiref_more_test.go new file mode 100644 index 0000000..bda28f2 --- /dev/null +++ b/multiref_more_test.go @@ -0,0 +1,131 @@ +package refbench + +import ( + "bytes" + "context" + "io" + "os/exec" + "strings" + "testing" + "time" +) + +func TestReadReportRejectsMutatedSchema2(t *testing.T) { + for name, mut := range map[string]func(*Row){ + "go accuracy": func(r *Row) { r.GoAccuracy.Coverage = 2 }, + "primary accuracy": func(r *Row) { r.PrimaryAccuracy.Points = -1 }, + "ref accuracy": func(r *Row) { r.Refs[0].Accuracy.Coverage = -1 }, + "missing ns": func(r *Row) { delete(r.Rounds[0].RefsNs, "boost:tgamma"); r.Rounds[0].RefsNs["x:y"] = 1 }, + "count": func(r *Row) { delete(r.Rounds[0].RefsNs, "boost:tgamma") }, + "short order": func(r *Row) { r.Rounds[0].Order = r.Rounds[0].Order[:2] }, + "duplicate order": func(r *Row) { r.Rounds[0].Order = []string{"go", "go", "primary", "boost:tgamma"} }, + "unknown order": func(r *Row) { r.Rounds[0].Order = []string{"go", "zz", "primary", "boost:tgamma"} }, + } { + rep, _ := runMulti(t, 4) + mut(&rep.Rows[0]) + var b bytes.Buffer + if err := rep.Write(&b); err != nil { + t.Fatalf("%s: %v", name, err) + } + if _, err := ReadReport(&b); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestRefNameEqualToIDIsOmitted(t *testing.T) { + c := Case{Name: "a", Func: "A", Go: func(int) {}, Parity: ParityNone, Refs: []Ref{{ID: "julia:x", Name: "julia:x", License: "MIT"}}} + if rows := c.refRows(); rows[0].Name != "" || rows[0].label() != "julia:x" { + t.Errorf("rows = %+v", rows) + } +} + +func TestSameRefIDs(t *testing.T) { + a := []RefRow{{ID: "a:x"}, {ID: "b:y"}} + if !sameRefIDs(a, a) || sameRefIDs(a, a[:1]) || sameRefIDs(a, []RefRow{{ID: "a:x"}, {ID: "b:z"}}) { + t.Error("sameRefIDs") + } +} + +func TestTableRowsWithDifferentReferences(t *testing.T) { + rep, _ := runMulti(t, 4) + other := rep.Rows[0] + other.Name = "other" + other.Refs = append([]RefRow(nil), other.Refs...) + other.Refs[0].ID = "fortran:amos" + other.Rounds = nil + rep.Rows = append(rep.Rows, other) + tab := string(Table(rep)) + if !strings.Contains(tab, "fortran:amos") || !strings.Contains(tab, "| — |") { + t.Errorf("table:\n%s", tab) + } + if sideName(Row{Reference: "scipy"}, SidePrimary) != "scipy" || sideName(Row{}, "zz:y") != "zz:y" { + t.Error("sideName") + } +} + +func TestEvalerError(t *testing.T) { + dir := t.TempDir() + path, rows, sha, _ := WriteF64(dir, "x", []float64{1}) + w := fake(t, dir, []string{"f"}, func(line string) string { + if strings.HasPrefix(line, "L") { + return "ok " + sha + } + return "err domain" + }) + if err := w.Load("x", path, 1, rows, sha); err != nil { + t.Fatal(err) + } + if _, err := w.Evaler("x", "f", dir+"/o", nil)(); err == nil { + t.Error("worker error lost") + } +} + +func TestStartRBW1PipeErrors(t *testing.T) { + old := execCommand + defer func() { execCommand = old }() + cfg := RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}} + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { + c := exec.CommandContext(ctx, "x") + c.Stdin = strings.NewReader("") + return c + } + if _, err := StartRBW1(context.Background(), cfg, nil, "x"); err == nil { + t.Error("stdin already set accepted") + } + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { + c := exec.CommandContext(ctx, "x") + c.Stdout = io.Discard + return c + } + if _, err := StartRBW1(context.Background(), cfg, nil, "x"); err == nil { + t.Error("stdout already set accepted") + } +} + +func TestRunStopsOnATimerErrorInARound(t *testing.T) { + var log []string + c := multiCase(&log) + calls := 0 + c.Refs[1].Timer = func(iters int) (time.Duration, error) { + if calls++; calls > 3 { // calibration takes 3 calls (warm-up, 1, then the scaled count): the failure is in a timed round + return 0, io.ErrUnexpectedEOF + } + return time.Duration(20000 * iters), nil + } + _, err := Run(context.Background(), []Case{c}, Options{Rounds: 4, MinTime: time.Millisecond, Load: func() (float64, error) { return 1, nil }}) + if err == nil || !strings.Contains(err.Error(), "unexpected EOF") { + t.Errorf("err = %v", err) + } +} + +func TestRunWithOnlyRetainsRefs(t *testing.T) { + var log []string + c := multiCase(&log) + c2 := multiCase(&log) + c2.Name = "other" + rep, err := Run(context.Background(), []Case{c, c2}, Options{Rounds: 4, MinTime: time.Millisecond, Only: map[string]bool{"other": true}, Load: func() (float64, error) { return 1, nil }}) + if err != nil || len(rep.Rows) != 1 || rep.Rows[0].Name != "other" || len(rep.Rows[0].Refs) != 2 { + t.Fatalf("%v %+v", err, rep) + } +} diff --git a/rbw1.go b/rbw1.go new file mode 100644 index 0000000..d0a1c0d --- /dev/null +++ b/rbw1.go @@ -0,0 +1,518 @@ +package refbench + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +// The rbw1 protocol: one request line, one reply line, plain ASCII, so that R +// base and Fortran (which have no JSON parser) can speak it without a pinned +// dependency. Python keeps its JSON protocol. +// +// L load float64 little-endian column-major -> ok +// W one untimed call (JIT warm-up, page-in) -> ok +// T time iters calls inside the worker -> ns +// E evaluate once, write rows float64 -> ok +// V versions and observed threading -> v k=v k=v ... +// any line on failure -> err +// +// Nothing the worker reads from the pipe is evaluated: is a name in a +// fixed table compiled or defined in the worker, must lie inside the +// run directory, and every size is bounded. Tokens are separated by one space; +// version values replace their spaces by underscores. +const ( + // RBW1MaxLine bounds a line, request or reply, without its newline. + RBW1MaxLine = 4096 + // RBW1MaxData bounds the bytes of one loaded argument file (256 MiB). + RBW1MaxData = 256 << 20 + // rbw1MaxDim bounds cols and rows of a request. + rbw1MaxDim = 1 << 28 + // rbw1MaxVersions bounds the pairs of a version reply. + rbw1MaxVersions = 256 +) + +// ErrRBW1LineTooLong is returned when a worker's line is longer than RBW1MaxLine. +var ErrRBW1LineTooLong = errors.New("rbw1: worker line longer than 4096 bytes") + +var ( + rbw1KeyPattern = regexp.MustCompile(`^[A-Za-z0-9_]{1,64}$`) + rbw1FnPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.]{0,63}$`) + rbw1SHAPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + rbw1VerKey = regexp.MustCompile(`^[A-Za-z0-9_./+-]{1,64}$`) +) + +// RBW1Reply is one parsed reply line. +type RBW1Reply struct { + // Kind is "ok", "ns" or "v". + Kind string + // Args are the tokens after "ok". + Args []string + // NS is the total nanoseconds of a "ns" reply. + NS float64 + // Versions are the pairs of a "v" reply. + Versions map[string]string +} + +func printableASCII(b []byte) bool { + for _, c := range b { + if c < 0x20 || c > 0x7e { + return false + } + } + return true +} + +// ParseRBW1Reply reads one worker line strictly: at most RBW1MaxLine printable +// ASCII bytes, tokens separated by exactly one space, a finite non-negative +// time in a "ns" reply, unique well-formed pairs in a "v" reply. An "err" line +// is an error. +func ParseRBW1Reply(line []byte) (RBW1Reply, error) { + var r RBW1Reply + switch { + case len(line) > RBW1MaxLine: + return r, ErrRBW1LineTooLong + case !printableASCII(line): + return r, errors.New("rbw1: reply is not printable ASCII") + } + s := string(line) + kind, rest, hasRest := strings.Cut(s, " ") + if kind == "err" { + return r, fmt.Errorf("reference failed: %s", rest) + } + var toks []string + if hasRest { + toks = strings.Split(rest, " ") + for _, t := range toks { + if t == "" { + return r, errors.New("rbw1: empty token (a doubled, leading or trailing space)") + } + } + } + r.Kind = kind + switch kind { + case "ok": + r.Args = toks + case "ns": + if len(toks) != 1 { + return RBW1Reply{}, errors.New("rbw1: ns reply needs one number") + } + f, err := strconv.ParseFloat(toks[0], 64) + if err != nil || !finiteNonNeg(f) { + return RBW1Reply{}, errors.New("rbw1: ns is not a finite non-negative number") + } + r.NS = f + case "v": + if len(toks) == 0 || len(toks) > rbw1MaxVersions { + return RBW1Reply{}, errors.New("rbw1: v reply needs 1 to 256 pairs") + } + r.Versions = make(map[string]string, len(toks)) + for _, t := range toks { + k, v, ok := strings.Cut(t, "=") + if !ok || !rbw1VerKey.MatchString(k) || v == "" { + return RBW1Reply{}, fmt.Errorf("rbw1: bad version pair %q", t) + } + if _, dup := r.Versions[k]; dup { + return RBW1Reply{}, fmt.Errorf("rbw1: version key %q twice", k) + } + r.Versions[k] = v + } + default: + return RBW1Reply{}, fmt.Errorf("rbw1: unknown reply %q", kind) + } + return r, nil +} + +// CheckSingleThread refuses a worker whose reply to V does not show one thread: +// the key "threads" must be present, and every key ending in "threads" +// (gc_threads, blas_threads, ...) must be 1. The environment variables that ask +// for one thread are not trusted, what the worker observed is. +func CheckSingleThread(v map[string]string) error { + if v["threads"] == "" { + return errors.New("rbw1: the worker does not report its thread count (key threads)") + } + for k, val := range v { + if (k == "threads" || strings.HasSuffix(k, "_threads")) && val != "1" { + return fmt.Errorf("rbw1: the worker runs with %s=%s, not 1: the timing would not be single-core", k, val) + } + } + return nil +} + +// NamespaceVersions prefixes the keys of a worker's versions with its kind +// ("julia/Bessels"), the form ref_versions records. +func NamespaceVersions(kind string, v map[string]string) map[string]string { + out := make(map[string]string, len(v)) + for k, val := range v { + out[kind+"/"+k] = val + } + return out +} + +// RBW1Config describes the run directory and the function table of a worker. +type RBW1Config struct { + // Dir is the absolute run directory: every path of a request lies inside it. + Dir string + // Funcs is the worker's function table. A request for another name is + // refused before it is sent. + Funcs []string +} + +// RBW1 is a long-lived worker speaking rbw1. +type RBW1 struct { + cmd *exec.Cmd + in io.WriteCloser + out *bufio.Scanner + dir string + funcs map[string]bool + rows map[string]int + warmed map[string]bool +} + +// NewRBW1 speaks rbw1 over a pair of streams (tests, or a process started by +// StartRBW1). +func NewRBW1(in io.WriteCloser, out io.Reader, cfg RBW1Config) (*RBW1, error) { + if !filepath.IsAbs(cfg.Dir) || filepath.Clean(cfg.Dir) != cfg.Dir || !printableASCII([]byte(cfg.Dir)) || strings.Contains(cfg.Dir, " ") { + return nil, fmt.Errorf("rbw1: run directory %q is not a clean absolute path without spaces", cfg.Dir) + } + if len(cfg.Funcs) == 0 { + return nil, errors.New("rbw1: no function table") + } + w := &RBW1{in: in, dir: cfg.Dir, funcs: map[string]bool{}, rows: map[string]int{}, warmed: map[string]bool{}} + for _, f := range cfg.Funcs { + if !rbw1FnPattern.MatchString(f) { + return nil, fmt.Errorf("rbw1: function name %q is not a plain identifier", f) + } + w.funcs[f] = true + } + sc := bufio.NewScanner(out) + // One byte more than a line and its newline: a longer line is refused + // without ever being buffered. + sc.Buffer(make([]byte, 0, RBW1MaxLine+2), RBW1MaxLine+2) + w.out = sc + return w, nil +} + +// StartRBW1 starts a worker process under env (nil: SingleThreadEnv) and speaks +// rbw1 with it. The run directory is passed to the worker as RBW1_DIR. The +// worker's stderr goes to the caller's. +func StartRBW1(ctx context.Context, cfg RBW1Config, env []string, name string, args ...string) (*RBW1, error) { + cmd := execCommand(ctx, name, args...) + if env == nil { + env = SingleThreadEnv() + } + cmd.Env = append(env, "RBW1_DIR="+cfg.Dir) + cmd.Stderr = os.Stderr + in, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + out, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + w, err := NewRBW1(in, out, cfg) + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + w.cmd = cmd + return w, nil +} + +func (w *RBW1) ask(parts ...string) (RBW1Reply, error) { + line := strings.Join(parts, " ") + if len(line) > RBW1MaxLine { + return RBW1Reply{}, errors.New("rbw1: request longer than 4096 bytes") + } + if _, err := io.WriteString(w.in, line+"\n"); err != nil { + return RBW1Reply{}, fmt.Errorf("worker: %w", err) + } + if !w.out.Scan() { + if err := w.out.Err(); err != nil { + if errors.Is(err, bufio.ErrTooLong) { + return RBW1Reply{}, ErrRBW1LineTooLong + } + return RBW1Reply{}, fmt.Errorf("worker: %w", err) + } + return RBW1Reply{}, errors.New("worker: process ended") + } + return ParseRBW1Reply(w.out.Bytes()) +} + +func (w *RBW1) check(key, fn string) error { + if !rbw1KeyPattern.MatchString(key) { + return fmt.Errorf("rbw1: key %q is not a plain identifier", key) + } + if !w.funcs[fn] { + return fmt.Errorf("rbw1: function %q is not in the worker's table", fn) + } + return nil +} + +// inDir reports whether path is a clean absolute path strictly inside the run +// directory, without spaces. +func (w *RBW1) inDir(path string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || !printableASCII([]byte(path)) || + strings.Contains(path, " ") || !strings.HasPrefix(path, w.dir+string(filepath.Separator)) { + return fmt.Errorf("rbw1: path %q is not a clean absolute path inside %s", path, w.dir) + } + return nil +} + +// Load asks the worker to read an argument file (cols columns of rows float64, +// column-major, little-endian) and compares the SHA-256 of the bytes it read +// with want. A mismatch is an error naming both digests: nothing is timed on +// data that did not arrive unaltered. +func (w *RBW1) Load(key, path string, cols, rows int, want string) error { + if !rbw1KeyPattern.MatchString(key) { + return fmt.Errorf("rbw1: key %q is not a plain identifier", key) + } + if err := w.inDir(path); err != nil { + return err + } + if cols <= 0 || rows <= 0 || cols > rbw1MaxDim || rows > rbw1MaxDim || int64(cols)*int64(rows)*8 > RBW1MaxData { + return fmt.Errorf("rbw1: %d x %d float64 is outside 1..%d bytes", cols, rows, RBW1MaxData) + } + r, err := w.ask("L", key, path, strconv.Itoa(cols), strconv.Itoa(rows)) + if err != nil { + return err + } + if r.Kind != "ok" || len(r.Args) != 1 || !rbw1SHAPattern.MatchString(r.Args[0]) { + return errors.New("rbw1: load reply is not 'ok '") + } + if r.Args[0] != want { + return fmt.Errorf("rbw1: the worker read %s, the Go side wrote %s: the inputs did not arrive unaltered", r.Args[0], want) + } + w.rows[key] = rows + return nil +} + +func (w *RBW1) okReply(parts ...string) (RBW1Reply, error) { + r, err := w.ask(parts...) + if err == nil && r.Kind != "ok" { + err = fmt.Errorf("rbw1: reply %q to %s, want ok", r.Kind, parts[0]) + } + return r, err +} + +// Warm asks for one untimed call of fn on the loaded key (JIT compilation, +// page-in). +func (w *RBW1) Warm(key, fn string) error { + if err := w.check(key, fn); err != nil { + return err + } + r, err := w.okReply("W", key, fn) + if err == nil && len(r.Args) != 0 { + err = errors.New("rbw1: warm-up reply has arguments") + } + return err +} + +// Time asks for iters calls of fn on the loaded key, timed inside the worker. +func (w *RBW1) Time(key, fn string, iters int) (time.Duration, error) { + if err := w.check(key, fn); err != nil { + return 0, err + } + if iters <= 0 { + return 0, errors.New("rbw1: iters must be positive") + } + r, err := w.ask("T", key, fn, strconv.Itoa(iters)) + if err != nil { + return 0, err + } + if r.Kind != "ns" { + return 0, fmt.Errorf("rbw1: reply %q to T, want ns", r.Kind) + } + if r.NS >= math.MaxInt64 { + return 0, errors.New("rbw1: ns overflows") + } + return time.Duration(r.NS), nil +} + +// Eval asks for one evaluation of fn on every point of the loaded key, written +// as rows float64 to outPath. It returns the number of non-finite values and of +// points the reference refused (domain errors, which the worker turns into NaN). +func (w *RBW1) Eval(key, fn, outPath string) (nonFinite, raised int, err error) { + if err := w.check(key, fn); err != nil { + return 0, 0, err + } + if err := w.inDir(outPath); err != nil { + return 0, 0, err + } + r, err := w.okReply("E", key, fn, outPath) + if err != nil { + return 0, 0, err + } + if len(r.Args) != 2 { + return 0, 0, errors.New("rbw1: eval reply is not 'ok '") + } + a, err1 := strconv.Atoi(r.Args[0]) + b, err2 := strconv.Atoi(r.Args[1]) + if err1 != nil || err2 != nil || a < 0 || b < 0 { + return 0, 0, errors.New("rbw1: eval counts are not non-negative integers") + } + return a, b, nil +} + +// Versions asks for the versions and the observed threading. +func (w *RBW1) Versions() (map[string]string, error) { + r, err := w.ask("V") + if err != nil { + return nil, err + } + if r.Kind != "v" { + return nil, fmt.Errorf("rbw1: reply %q to V, want v", r.Kind) + } + return r.Versions, nil +} + +// Timer returns a Timer of fn on the loaded key. The first call is preceded by +// one untimed W, so that the interval of the first timed call is not the +// compilation of the kernel. +func (w *RBW1) Timer(key, fn string) Timer { + return func(iters int) (time.Duration, error) { + if id := key + "/" + fn; !w.warmed[id] { + if err := w.Warm(key, fn); err != nil { + return 0, err + } + w.warmed[id] = true + } + return w.Time(key, fn, iters) + } +} + +// Evaler returns an EvalFunc of fn on the loaded key: the worker writes its +// values to outPath, which is read back (and must hold exactly one value per +// row). The counts of non-finite values and refused points are reported to +// count when it is not nil. +func (w *RBW1) Evaler(key, fn, outPath string, count func(nonFinite, raised int)) EvalFunc { + return func() ([]float64, error) { + rows, ok := w.rows[key] + if !ok { + return nil, fmt.Errorf("rbw1: key %q was not loaded", key) + } + nf, rs, err := w.Eval(key, fn, outPath) + if err != nil { + return nil, err + } + if count != nil { + count(nf, rs) + } + return ReadF64(outPath, rows) + } +} + +// Close closes the worker's input and waits for the process, if there is one. +func (w *RBW1) Close() error { + err := w.in.Close() + if w.cmd != nil { + return w.cmd.Wait() + } + return err +} + +// Argument files. + +// userCacheDir is replaceable by tests. +var userCacheDir = os.UserCacheDir + +// DataDir returns (and creates) the run directory of a repository's commit: +// os.UserCacheDir()/refbench//. It belongs to the tool: not to +// the repository, not to /tmp. repo may contain one "/" ("go-x/y"). +func DataDir(repo, commit string) (string, error) { + for _, s := range []string{repo, commit} { + if s == "" || s == "." || strings.Contains(s, "..") || strings.ContainsAny(s, " \\\x00") || !printableASCII([]byte(s)) { + return "", fmt.Errorf("data dir: %q is not a plain name", s) + } + } + base, err := userCacheDir() + if err != nil { + return "", err + } + dir := filepath.Join(base, "refbench", filepath.FromSlash(repo), commit) + for _, e := range strings.Split(filepath.ToSlash(dir), "/") { + if e == ".git" { + return "", fmt.Errorf("data dir %s is inside a .git directory", dir) + } + } + if strings.Contains(dir, " ") { + return "", fmt.Errorf("data dir %s contains a space (rbw1 tokens are space separated)", dir) + } + return dir, os.MkdirAll(dir, 0o700) +} + +// dataLimit is RBW1MaxData; tests lower it to reach the refusal without +// allocating 256 MiB. +var dataLimit int64 = RBW1MaxData + +// WriteF64 writes cols (equal lengths) as float64 little-endian column-major to +// dir/name and returns the path, the number of rows and the SHA-256 of the +// bytes: what Load compares with the worker's. A file over RBW1MaxData is +// refused. +func WriteF64(dir, name string, cols ...[]float64) (path string, rows int, sha string, err error) { + if !rbw1KeyPattern.MatchString(name) { + return "", 0, "", fmt.Errorf("argument file name %q is not a plain identifier", name) + } + if len(cols) == 0 || len(cols[0]) == 0 { + return "", 0, "", errors.New("no data to write") + } + rows = len(cols[0]) + if int64(len(cols))*int64(rows)*8 > dataLimit { + return "", 0, "", fmt.Errorf("argument file larger than %d bytes", dataLimit) + } + buf := make([]byte, 0, len(cols)*rows*8) + for _, c := range cols { + if len(c) != rows { + return "", 0, "", errors.New("columns of different lengths") + } + for _, x := range c { + buf = binary.LittleEndian.AppendUint64(buf, math.Float64bits(x)) + } + } + path = filepath.Join(dir, name+".f64") + if err := os.WriteFile(path, buf, 0o600); err != nil { + return "", 0, "", err + } + sum := sha256.Sum256(buf) + return path, rows, hex.EncodeToString(sum[:]), nil +} + +// ReadF64 reads exactly n float64 little-endian values from path. +func ReadF64(path string, n int) ([]float64, error) { + if n < 0 || int64(n)*8 > dataLimit { + return nil, errors.New("rbw1: result size out of range") + } + // The size is checked before the read: a worker cannot make us buffer more. + var b []byte + st, err := os.Stat(path) + if err == nil && st.Size() != int64(n)*8 { + err = fmt.Errorf("rbw1: %s holds %d bytes, want %d", path, st.Size(), int64(n)*8) + } + if err == nil { + b, err = os.ReadFile(path) + } + if err != nil { + return nil, err + } + out := make([]float64, n) + for i := range out { + out[i] = math.Float64frombits(binary.LittleEndian.Uint64(b[8*i:])) + } + return out, nil +} diff --git a/rbw1_test.go b/rbw1_test.go new file mode 100644 index 0000000..921465f --- /dev/null +++ b/rbw1_test.go @@ -0,0 +1,733 @@ +package refbench + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "runtime" + "runtime/debug" + "strings" + "testing" + "time" +) + +func TestParseRBW1Reply(t *testing.T) { + ok := map[string]RBW1Reply{ + "ok": {Kind: "ok"}, + "ok abc": {Kind: "ok", Args: []string{"abc"}}, + "ok 1 2": {Kind: "ok", Args: []string{"1", "2"}}, + "ns 12.5": {Kind: "ns", NS: 12.5}, + "ns 0": {Kind: "ns"}, + "v julia=1.13.3 a/b+c=x@y": {Kind: "v", Versions: map[string]string{"julia": "1.13.3", "a/b+c": "x@y"}}, + "v threads=1": {Kind: "v", Versions: map[string]string{"threads": "1"}}, + "ok " + strings.Repeat("a", 64): {Kind: "ok", Args: []string{strings.Repeat("a", 64)}}, + } + for in, want := range ok { + got, err := ParseRBW1Reply([]byte(in)) + if err != nil || got.Kind != want.Kind || got.NS != want.NS || len(got.Args) != len(want.Args) || len(got.Versions) != len(want.Versions) { + t.Errorf("%q = %+v, %v; want %+v", in, got, err, want) + } + for k, v := range want.Versions { + if got.Versions[k] != v { + t.Errorf("%q: %s = %q", in, k, got.Versions[k]) + } + } + } + bad := []string{ + ``, `err boom`, `err`, `ok 1`, `ok 1 `, ` ok`, `ns`, `ns 1 2`, `ns -1`, `ns nan`, `ns +Inf`, `ns 1e999`, `ns x`, + `v`, `v a`, `v =x`, `v a=`, `v a=1 a=2`, `v bad key=1`, `v a b=1`, `nope 1`, "ok \x00", "ok café", "ok \t1", "ok \x7f", + "v " + strings.Repeat("k=1 ", 300), + } + for _, in := range bad { + if _, err := ParseRBW1Reply([]byte(in)); err == nil { + t.Errorf("%q accepted", in) + } + } + if _, err := ParseRBW1Reply([]byte("err boom")); err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("err reply: %v", err) + } + if _, err := ParseRBW1Reply([]byte("ok " + strings.Repeat("x", RBW1MaxLine))); !errors.Is(err, ErrRBW1LineTooLong) { + t.Errorf("oversized: %v", err) + } + if _, err := ParseRBW1Reply([]byte("ok " + strings.Repeat("x", RBW1MaxLine-3))); err != nil { + t.Errorf("a line of exactly 4096 bytes refused: %v", err) + } +} + +func FuzzParseRBW1Reply(f *testing.F) { + for _, s := range []string{"ok", "ok " + strings.Repeat("0", 64), "ns 12.5", "ns 1e999", "v julia=1.13.3 threads=1", "err boom", "ok 1 2", "v a=1 a=2", "ok x", "ns -0"} { + f.Add([]byte(s)) + } + f.Fuzz(func(t *testing.T, b []byte) { + r, err := ParseRBW1Reply(b) + if err != nil { + return + } + if len(b) > RBW1MaxLine || !printableASCII(b) { + t.Fatalf("accepted %q", b) + } + if r.Kind == "ns" && !finiteNonNeg(r.NS) { + t.Fatalf("accepted ns %v", r.NS) + } + if r.Kind == "v" && len(r.Versions) > rbw1MaxVersions { + t.Fatal("too many versions") + } + if len(r.Args) > 0 && strings.Contains(strings.Join(r.Args, ""), " ") { + t.Fatal("space in a token") + } + }) +} + +// fake is an in-process worker: respond maps a request line to a reply line. +func fake(t *testing.T, dir string, funcs []string, respond func(line string) string) *RBW1 { + t.Helper() + inR, inW := io.Pipe() + outR, outW := io.Pipe() + go func() { + sc := bufio.NewScanner(inR) + for sc.Scan() { + fmt.Fprintln(outW, respond(sc.Text())) + } + outW.Close() + }() + w, err := NewRBW1(inW, outR, RBW1Config{Dir: dir, Funcs: funcs}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { inW.Close(); outR.Close() }) + return w +} + +func TestRBW1Requests(t *testing.T) { + dir := t.TempDir() + path, rows, sha, err := WriteF64(dir, "x", []float64{1, 2, 3}, []float64{4, 5, 6}) + if err != nil || rows != 3 { + t.Fatal(err) + } + var calls []string + w := fake(t, dir, []string{"gamma", "k.v2"}, func(line string) string { + calls = append(calls, line) + f := strings.Fields(line) + switch f[0] { + case "L": + return "ok " + sha + case "W": + return "ok" + case "T": + return "ns " + f[3] + "000" + case "E": + os.WriteFile(f[3], make([]byte, 24), 0o600) + return "ok 1 2" + case "V": + return "v threads=1" + } + return "err unknown" + }) + if err := w.Load("x", path, 2, 3, sha); err != nil { + t.Fatal(err) + } + tm := w.Timer("x", "gamma") + d, err := tm(5) + if err != nil || d != 5000 { + t.Fatalf("%v %v", d, err) + } + if _, err := tm(1); err != nil { + t.Fatal(err) + } + warmups := 0 + for _, c := range calls { + if strings.HasPrefix(c, "W ") { + warmups++ + } + } + if warmups != 1 { + t.Errorf("%d warm-ups, want one before the first timed call: %v", warmups, calls) + } + out := filepath.Join(dir, "out.f64") + nf, raised, err := w.Eval("x", "gamma", out) + if err != nil || nf != 1 || raised != 2 { + t.Errorf("eval = %d %d %v", nf, raised, err) + } + var seen [2]int + vals, err := w.Evaler("x", "k.v2", out, func(a, b int) { seen = [2]int{a, b} })() + if err != nil || len(vals) != 3 || seen != [2]int{1, 2} { + t.Errorf("evaler = %v %v %v", vals, seen, err) + } + if v, err := w.Versions(); err != nil || v["threads"] != "1" { + t.Errorf("versions = %v %v", v, err) + } +} + +func TestRBW1RefusesBeforeSending(t *testing.T) { + dir := t.TempDir() + sent := 0 + w := fake(t, dir, []string{"f"}, func(string) string { sent++; return "ok" }) + sha := strings.Repeat("0", 64) + for name, err := range map[string]error{ + "fn not in table": w.Warm("x", "g"), + "fn injection": w.Warm("x", "f; rm -rf /"), + "bad key": w.Warm("x y", "f"), + "relative path": w.Load("x", "rel.f64", 1, 1, sha), + "outside": w.Load("x", filepath.Join(filepath.Dir(dir), "o.f64"), 1, 1, sha), + "dotdot": w.Load("x", dir+"/../o.f64", 1, 1, sha), + "the dir itself": w.Load("x", dir, 1, 1, sha), + "space in path": w.Load("x", dir+"/a b", 1, 1, sha), + "zero cols": w.Load("x", dir+"/a", 0, 1, sha), + "huge": w.Load("x", dir+"/a", 1<<20, 1<<20, sha), + "overflow 32 bits": w.Load("x", dir+"/a", 1<<28, 1<<28, sha), + "bad load key": w.Load("", dir+"/a", 1, 1, sha), + "eval outside": evalErr(w, "/etc/passwd"), + "time bad fn": timeErr(w, "g", 1), + "time zero iters": timeErr(w, "f", 0), + "eval bad fn": evalFnErr(w, dir), + "unloaded evaler": evalerErr(w, dir), + "warm in timer": timerErr(w, "g"), + "huge request line": w.Load("x", dir+"/"+strings.Repeat("a", RBW1MaxLine), 1, 1, sha), + } { + if err == nil { + t.Errorf("%s: accepted", name) + } + } + if sent != 0 { + t.Errorf("%d requests reached the worker", sent) + } +} + +func evalErr(w *RBW1, p string) error { _, _, err := w.Eval("x", "f", p); return err } +func timeErr(w *RBW1, fn string, it int) error { + _, err := w.Time("x", fn, it) + return err +} +func evalFnErr(w *RBW1, dir string) error { _, _, err := w.Eval("x", "zz", dir+"/o"); return err } +func evalerErr(w *RBW1, dir string) error { + _, err := w.Evaler("never", "f", dir+"/o", nil)() + return err +} +func timerErr(w *RBW1, fn string) error { _, err := w.Timer("x", fn)(1); return err } + +func TestRBW1RefusesBadReplies(t *testing.T) { + dir := t.TempDir() + sha := strings.Repeat("0", 64) + reply := "" + w := fake(t, dir, []string{"f"}, func(string) string { return reply }) + check := func(name string, err error) { + t.Helper() + if err == nil { + t.Errorf("%s: accepted %q", name, reply) + } + } + for _, r := range []string{"ns 1", "ok", "ok zz", "ok " + strings.Repeat("A", 64), "ok " + strings.Repeat("1", 64), "err no"} { + reply = r + check("load", w.Load("x", dir+"/a", 1, 1, sha)) + } + for _, r := range []string{"ns 1", "ok 1", "err no", "v a=1"} { + reply = r + check("warm", w.Warm("x", "f")) + } + for _, r := range []string{"ok", "v a=1", "ns 1e300", "err x"} { + reply = r + _, err := w.Time("x", "f", 1) + check("time", err) + } + for _, r := range []string{"ns 1", "ok", "ok 1", "ok -1 0", "ok 0 x", "ok 1 2 3", "err x"} { + reply = r + _, _, err := w.Eval("x", "f", dir+"/o") + check("eval", err) + } + for _, r := range []string{"ok", "ns 1", "err x"} { + reply = r + _, err := w.Versions() + check("versions", err) + } +} + +func TestRBW1LoadDetectsAnAlteredInput(t *testing.T) { + // Negative control: a worker that perturbs one value of its input reads + // another digest, and Load refuses it, naming both. + dir := t.TempDir() + path, _, sha, err := WriteF64(dir, "x", []float64{1, 2, 3}) + if err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(path) + b[8] ^= 1 // one bit of the second value + altered := filepath.Join(dir, "y.f64") + os.WriteFile(altered, b, 0o600) + _, _, shaAltered, _ := WriteF64(dir, "z", []float64{1, math.Float64frombits(math.Float64bits(2) ^ 1), 3}) + w := fake(t, dir, []string{"f"}, func(string) string { return "ok " + shaAltered }) + err = w.Load("x", path, 1, 3, sha) + if err == nil || !strings.Contains(err.Error(), sha) || !strings.Contains(err.Error(), shaAltered) { + t.Errorf("err = %v", err) + } +} + +func TestRBW1ProcessEnded(t *testing.T) { + inR, inW := io.Pipe() + outR, outW := io.Pipe() + go func() { io.Copy(io.Discard, inR); outW.Close() }() + w, err := NewRBW1(inW, outR, RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}) + if err != nil { + t.Fatal(err) + } + inW.Close() // the write fails + if err := w.Warm("x", "f"); err == nil { + t.Error("write to a closed worker accepted") + } + if err := w.Close(); err != nil { + t.Error(err) + } + // EOF on the reply stream. + in2R, in2W := io.Pipe() + out2R, out2W := io.Pipe() + go func() { io.Copy(io.Discard, in2R) }() + out2W.Close() + w2, _ := NewRBW1(in2W, out2R, RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}) + if err := w2.Warm("x", "f"); err == nil || !strings.Contains(err.Error(), "ended") { + t.Errorf("eof: %v", err) + } + // A reader that fails. + in3R, in3W := io.Pipe() + go func() { io.Copy(io.Discard, in3R) }() + w3, _ := NewRBW1(in3W, errReader{}, RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}) + if err := w3.Warm("x", "f"); err == nil || !strings.Contains(err.Error(), "io") { + t.Errorf("read error: %v", err) + } +} + +type errReader struct{} + +func (errReader) Read([]byte) (int, error) { return 0, errors.New("io") } + +func TestNewRBW1Refuses(t *testing.T) { + good := t.TempDir() + for name, cfg := range map[string]RBW1Config{ + "relative": {Dir: "rel", Funcs: []string{"f"}}, + "unclean": {Dir: good + "/./x", Funcs: []string{"f"}}, + "space": {Dir: good + "/a b", Funcs: []string{"f"}}, + "no funcs": {Dir: good}, + "bad func": {Dir: good, Funcs: []string{"1f"}}, + "func a b": {Dir: good, Funcs: []string{"a b"}}, + "non ascii": {Dir: good + "/é", Funcs: []string{"f"}}, + } { + if _, err := NewRBW1(nopCloser{}, strings.NewReader(""), cfg); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +type nopCloser struct{} + +func (nopCloser) Write(p []byte) (int, error) { return len(p), nil } +func (nopCloser) Close() error { return nil } + +func TestBombOversizedLine(t *testing.T) { + // A worker that streams 256 MiB without a newline must be refused after + // 4 KiB, under a memory limit, without the heap growing. + old := debug.SetMemoryLimit(128 << 20) + defer debug.SetMemoryLimit(old) + outR, outW := io.Pipe() + go func() { + chunk := []byte(strings.Repeat("x", 1<<20)) + for i := 0; i < 256; i++ { + if _, err := outW.Write(chunk); err != nil { + return + } + } + outW.Close() + }() + w, err := NewRBW1(nopCloser{}, outR, RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}) + if err != nil { + t.Fatal(err) + } + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + err = w.Warm("x", "f") + outR.Close() + runtime.ReadMemStats(&after) + if !errors.Is(err, ErrRBW1LineTooLong) { + t.Fatalf("err = %v, want a refusal", err) + } + if grown := int64(after.HeapAlloc) - int64(before.HeapAlloc); grown > 8<<20 { + t.Errorf("heap grew by %d bytes", grown) + } + // And a line of 1 MiB handed to the parser. + if _, err := ParseRBW1Reply([]byte(strings.Repeat("x", 1<<20))); !errors.Is(err, ErrRBW1LineTooLong) { + t.Errorf("parser: %v", err) + } +} + +// TestHelperRBW1 is the fake rbw1 worker process the tests below start. +func TestHelperRBW1(t *testing.T) { + mode := os.Getenv("REFBENCH_RBW1_HELPER") + if mode == "" { + return + } + if mode == "bomb" { + // 64 MiB without a newline, under the memory limit the test sets. + chunk := []byte(strings.Repeat("x", 1<<20)) + for i := 0; i < 64; i++ { + if _, err := os.Stdout.Write(chunk); err != nil { + os.Exit(0) + } + } + os.Exit(0) + } + threads, julia := "1", "1.13.3" + switch mode { + case "threads": + threads = "4" + case "wrongjulia": + julia = "1.12.0" + } + sc := bufio.NewScanner(os.Stdin) + for sc.Scan() { + switch { + case mode == "die": + os.Exit(0) + case mode == "noversion": + fmt.Println("ok") + case sc.Text() == "V": + fmt.Printf("v julia=%s threads=%s gc_threads=1 blas_threads=1 Bessels=0.2.8@abc dir=%s\n", julia, threads, os.Getenv("RBW1_DIR")) + default: + fmt.Println("ok") + } + } + os.Exit(0) +} + +func helperRBW1(t *testing.T, mode string) { + t.Helper() + t.Setenv("REFBENCH_RBW1_HELPER", mode) + old := execCommand + t.Cleanup(func() { execCommand = old }) + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { + return exec.CommandContext(ctx, os.Args[0], "-test.run=TestHelperRBW1") + } +} + +func juliaCfg(t *testing.T) JuliaConfig { + return JuliaConfig{Pkgx: "pkgx", Version: "1.13.3", Project: "/p", Kernels: "/k.jl", Depot: "/d", Dir: t.TempDir(), Funcs: []string{"gamma"}} +} + +func TestStartJulia(t *testing.T) { + helperRBW1(t, "ok") + c := juliaCfg(t) + w, v, err := StartJulia(context.Background(), c) + if err != nil { + t.Fatal(err) + } + defer w.Close() + if v["julia/julia"] != "1.13.3" || v["julia/Bessels"] != "0.2.8@abc" || v["julia/dir"] != c.Dir { + t.Errorf("versions = %v", v) + } + if err := w.Warm("x", "gamma"); err != nil { + t.Error(err) + } +} + +func TestStartJuliaRefuses(t *testing.T) { + for mode, want := range map[string]string{"threads": "threads=4", "wrongjulia": "pin", "noversion": "want v", "die": "ended"} { + helperRBW1(t, mode) + _, _, err := StartJulia(context.Background(), juliaCfg(t)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: %v", mode, err) + } + } + // A bad configuration is refused before a process starts. + c := juliaCfg(t) + c.Dir = "rel" + if _, _, err := StartJulia(context.Background(), c); err == nil { + t.Error("relative dir accepted") + } + // And a process that cannot start. + old := execCommand + defer func() { execCommand = old }() + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { + return exec.CommandContext(ctx, "/nonexistent/refbench-test") + } + if _, _, err := StartJulia(context.Background(), juliaCfg(t)); err == nil { + t.Error("missing executable accepted") + } +} + +func TestBombOversizedLineFromAProcess(t *testing.T) { + // The process writes 64 MiB without a newline; the child runs under a + // GOMEMLIMIT of its own and the parent under the limit of the previous test. + helperRBW1(t, "bomb") + t.Setenv("GOMEMLIMIT", "64MiB") + old := debug.SetMemoryLimit(128 << 20) + defer debug.SetMemoryLimit(old) + w, err := StartRBW1(context.Background(), RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}, os.Environ(), "x") + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- w.Warm("x", "f") }() + select { + case err := <-done: + if !errors.Is(err, ErrRBW1LineTooLong) { + t.Errorf("err = %v, want a refusal", err) + } + case <-time.After(30 * time.Second): + t.Fatal("no refusal") + } + w.cmd.Process.Kill() + _ = w.Close() +} + +func TestJuliaArgsAndEnv(t *testing.T) { + c := juliaCfg(t) + args := JuliaArgs(c) + want := []string{"+julialang.org@=1.13.3", "--", "julia", "--startup-file=no", "--history-file=no", "-t", "1", "--project=/p", "-e"} + for i, w := range want { + if args[i] != w { + t.Errorf("args[%d] = %q, want %q", i, args[i], w) + } + } + if args[len(want)] != JuliaWorkerSource || args[len(args)-1] != "/k.jl" || !strings.Contains(JuliaWorkerSource, "RBW1_DIR") { + t.Error("the embedded worker or the kernels file is not passed") + } + env := strings.Join(JuliaEnv(c), "\n") + for _, e := range []string{"JULIA_NUM_THREADS=1", "JULIA_NUM_GC_THREADS=1", "JULIA_DEPOT_PATH=/d", "JULIA_PKG_OFFLINE=true", "OMP_NUM_THREADS=1"} { + if !strings.Contains(env, e) { + t.Errorf("env lacks %s", e) + } + } + if !strings.Contains(CompiledHarnessSource, "rbw1_kernels") { + t.Error("harness not embedded") + } +} + +func TestCheckSingleThreadAndNamespace(t *testing.T) { + good := map[string]string{"threads": "1", "gc_threads": "1", "blas_threads": "1", "julia": "1.13.3"} + if err := CheckSingleThread(good); err != nil { + t.Error(err) + } + for name, v := range map[string]map[string]string{ + "missing": {"julia": "1"}, + "threads": {"threads": "2"}, + "gc": {"threads": "1", "gc_threads": "4"}, + "blas": {"threads": "1", "blas_threads": "8"}, + "empty": {}, + } { + if CheckSingleThread(v) == nil { + t.Errorf("%s accepted", name) + } + } + ns := NamespaceVersions("julia", good) + if ns["julia/julia"] != "1.13.3" || len(ns) != len(good) { + t.Errorf("ns = %v", ns) + } +} + +func TestDataDirAndFiles(t *testing.T) { + base := t.TempDir() + old := userCacheDir + defer func() { userCacheDir = old }() + userCacheDir = func() (string, error) { return base, nil } + dir, err := DataDir("go-numerics/special", strings.Repeat("a", 40)) + if err != nil || !strings.HasPrefix(dir, filepath.Join(base, "refbench", "go-numerics", "special")) { + t.Fatalf("dir = %q, %v", dir, err) + } + if st, err := os.Stat(dir); err != nil || !st.IsDir() { + t.Error("not created") + } + for _, c := range [][2]string{{"", "a"}, {"a", ""}, {"..", "a"}, {"a/../b", "c"}, {"a", "b c"}, {"a\\b", "c"}, {".", "c"}, {"é", "c"}} { + if _, err := DataDir(c[0], c[1]); err == nil { + t.Errorf("DataDir(%q, %q) accepted", c[0], c[1]) + } + } + userCacheDir = func() (string, error) { return filepath.Join(base, ".git"), nil } + if _, err := DataDir("a", "b"); err == nil { + t.Error("a .git directory accepted") + } + userCacheDir = func() (string, error) { return base + "/sp ace", nil } + if _, err := DataDir("a", "b"); err == nil { + t.Error("a space accepted") + } + userCacheDir = func() (string, error) { return "", errors.New("no home") } + if _, err := DataDir("a", "b"); err == nil { + t.Error("cache dir error lost") + } + + // WriteF64 / ReadF64 round trip, little-endian column-major. + p, rows, sha, err := WriteF64(dir, "args", []float64{1, 2}, []float64{3, 4}) + if err != nil || rows != 2 || len(sha) != 64 { + t.Fatal(err) + } + b, _ := os.ReadFile(p) + if len(b) != 32 || b[7] != 0x3f || b[15] != 0x40 { // 1.0 and 2.0, little-endian + t.Errorf("bytes = %x", b) + } + got, err := ReadF64(p, 4) + if err != nil || got[0] != 1 || got[3] != 4 { + t.Errorf("read = %v %v", got, err) + } + if _, err := ReadF64(p, 3); err == nil { + t.Error("wrong size accepted") + } + if _, err := ReadF64(p+".missing", 1); err == nil { + t.Error("missing file accepted") + } + if _, err := ReadF64(p, -1); err == nil { + t.Error("negative n accepted") + } + oldLimit := dataLimit + dataLimit = 16 + defer func() { dataLimit = oldLimit }() + if _, err := ReadF64(p, 3); err == nil || !strings.Contains(err.Error(), "out of range") { + t.Errorf("an n over the limit accepted: %v", err) + } + if _, _, _, err := WriteF64(dir, "big", []float64{1, 2, 3}); err == nil { + t.Error("a file over the limit accepted") + } + dataLimit = oldLimit + for name, f := range map[string]func() error{ + "bad name": func() error { _, _, _, e := WriteF64(dir, "a/b", []float64{1}); return e }, + "no data": func() error { _, _, _, e := WriteF64(dir, "a"); return e }, + "empty": func() error { _, _, _, e := WriteF64(dir, "a", nil); return e }, + "ragged": func() error { _, _, _, e := WriteF64(dir, "a", []float64{1}, []float64{1, 2}); return e }, + "no dir": func() error { _, _, _, e := WriteF64(dir+"/missing", "a", []float64{1}); return e }, + } { + if f() == nil { + t.Errorf("%s accepted", name) + } + } +} + +func TestBuildWorkerRecordsSourcesAndFlags(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "a.cpp") + os.WriteFile(src, []byte("int main(){}\n"), 0o644) + t.Setenv("REFBENCH_HELPER", "1") + old := execCommand + defer func() { execCommand = old }() + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { return helperCommand(ctx, "cc", "ok") } + rec, err := BuildWorker(context.Background(), "c++", filepath.Join(dir, "w"), ReleaseFlags, []string{src, "-lgfortran"}) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(rec["src/a.cpp"], "sha256:") || rec["flags"] != "-O2 -lgfortran" { + t.Errorf("record = %v", rec) + } + if _, err := BuildWorker(context.Background(), "c++", "w", nil, nil); err == nil { + t.Error("no sources accepted") + } + if _, err := BuildWorker(context.Background(), "c++", "w", nil, []string{filepath.Join(dir, "missing.cpp")}); err == nil { + t.Error("missing source accepted") + } + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { return helperCommand(ctx, "cc", "fail") } + if _, err := BuildWorker(context.Background(), "c++", filepath.Join(dir, "w"), nil, []string{src}); err == nil { + t.Error("compiler failure lost") + } +} + +// TestCompiledWorkers builds the real harness with the system C++ compiler (and +// the Fortran example with gfortran) and speaks rbw1 with it. Skipped without a +// compiler: the CI of a repository never needs one. +func TestCompiledWorkers(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the harness is built and run on POSIX hosts") + } + cxx, err := exec.LookPath("c++") + if err != nil { + t.Skip("no c++ compiler") + } + work := t.TempDir() + harness := filepath.Join(work, "rbw1.cpp") + os.WriteFile(harness, []byte(CompiledHarnessSource), 0o644) + bin := filepath.Join(work, "worker-cpp") + rec, err := BuildWorker(context.Background(), cxx, bin, []string{"-O2", "-std=c++17"}, []string{harness, "workers/compiled/examples/cpp/kernels.cpp"}) + if err != nil { + t.Fatal(err) + } + if rec["src/kernels.cpp"] == "" { + t.Error("no digest recorded") + } + exerciseCompiled(t, bin, "sqrt") + + gf, err := exec.LookPath("gfortran") + if err != nil { + t.Log("no gfortran: Fortran example not built") + return + } + obj := filepath.Join(work, "sqrt.o") + if out, err := exec.Command(gf, "-O2", "-c", "-o", obj, "workers/compiled/examples/fortran/sqrt.f90").CombinedOutput(); err != nil { + t.Fatalf("%v\n%s", err, out) + } + binF := filepath.Join(work, "worker-f90") + if _, err := BuildWorker(context.Background(), cxx, binF, []string{"-O2", "-std=c++17"}, []string{harness, "workers/compiled/examples/fortran/table.cpp", obj, "-lgfortran"}); err != nil { + t.Skipf("cannot link the Fortran example here: %v", err) + } + exerciseCompiled(t, binF, "sqrt") +} + +func exerciseCompiled(t *testing.T, bin, fn string) { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + w, err := StartRBW1(context.Background(), RBW1Config{Dir: dir, Funcs: []string{fn}}, nil, bin) + if err != nil { + t.Fatal(err) + } + defer w.Close() + v, err := w.Versions() + if err != nil || CheckSingleThread(v) != nil || v["compiler"] == "" { + t.Fatalf("versions = %v %v", v, err) + } + x := []float64{1, 4, 9, -1, 16} + path, rows, sha, err := WriteF64(dir, "x", x) + if err != nil { + t.Fatal(err) + } + if err := w.Load("x", path, 1, rows, sha); err != nil { + t.Fatal(err) + } + // Negative control: a planted wrong digest is refused with both digests. + if err := w.Load("x", path, 1, rows, strings.Repeat("0", 64)); err == nil || !strings.Contains(err.Error(), sha) { + t.Errorf("wrong digest: %v", err) + } + var nf int + got, err := w.Evaler("x", fn, filepath.Join(dir, "o.f64"), func(n, _ int) { nf = n })() + if err != nil { + t.Fatal(err) + } + for i, want := range []float64{1, 2, 3, math.NaN(), 4} { + if got[i] != want && !(math.IsNaN(got[i]) && math.IsNaN(want)) { + t.Errorf("sqrt(%v) = %v, want %v", x[i], got[i], want) + } + } + if nf != 1 { + t.Errorf("non-finite = %d, want 1", nf) + } + d, err := w.Timer("x", fn)(1000) + if err != nil || d <= 0 { + t.Errorf("time = %v %v", d, err) + } + if _, err := Calibrate(w.Timer("x", fn), time.Millisecond); err != nil { + t.Errorf("calibrate: %v", err) + } + // The worker refuses what the Go side would not send: a path outside the + // run directory, a function outside its table, a request that is not ours. + for _, line := range []string{"L x /etc/passwd 1 1", "T x nofn 1", "W nokey " + fn, "E x " + fn + " /tmp/x", "zz", "L x " + path + " 1", "T x " + fn + " 0", "T x " + fn + " 1 1"} { + if _, err := w.ask(strings.Split(line, " ")...); err == nil { + t.Errorf("worker accepted %q", line) + } + } + if _, err := w.ask(strings.Repeat("x", 4100)); err == nil { + t.Error("oversized request not refused by the client") + } + // The harness itself refuses a line over 4096 bytes (bypassing the client). + io.WriteString(w.in, strings.Repeat("x", 5000)+"\n") + if !w.out.Scan() || !strings.HasPrefix(w.out.Text(), "err ") { + t.Errorf("harness reply to an oversized line: %q", w.out.Text()) + } +} diff --git a/runner.go b/runner.go index 238834b..096f390 100644 --- a/runner.go +++ b/runner.go @@ -259,7 +259,9 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { if err != nil { return row, err } - rd := Round{LoadBefore: load, GoFirst: sideIndex(sides, r%k, SideGo) < sideIndex(sides, r%k, SidePrimary)} + // Go is side 0 and the primary side 1: at position (i - r mod k) mod k. + goPos, primaryPos := (k-r%k)%k, (k+1-r%k)%k + rd := Round{LoadBefore: load, GoFirst: goPos < primaryPos} if k > 2 { rd.RefsNs = make(map[string]float64, k-2) } @@ -306,17 +308,6 @@ type roundSide struct { iters int } -// sideIndex returns the position of side id in the round whose rotation is rot. -func sideIndex(sides []roundSide, rot int, id string) int { - k := len(sides) - for p := 0; p < k; p++ { - if sides[(rot+p)%k].id == id { - return p - } - } - return -1 -} - // scoreAccuracy evaluates Go, the primary reference and every Ref with an Eval // on the case's points and stores the results in the row. It is deterministic, // so it is not gated; an evaluation error stops the run (a series with a hole diff --git a/table.go b/table.go index abac8a5..79b63dd 100644 --- a/table.go +++ b/table.go @@ -69,7 +69,7 @@ func refsSection(b *bytes.Buffer, r *Report) { for _, ref := range row.Refs { if _, ok := names[ref.ID]; !ok { ids = append(ids, ref.ID) - names[ref.ID] = ref.label() + names[ref.ID] = ref.ID // the ID is unambiguous; the best column shows the name } } } diff --git a/testdata/fuzz/FuzzParseRBW1Reply/seed-doubled-space b/testdata/fuzz/FuzzParseRBW1Reply/seed-doubled-space new file mode 100644 index 0000000..b83f76f --- /dev/null +++ b/testdata/fuzz/FuzzParseRBW1Reply/seed-doubled-space @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("ok 1 2") diff --git a/testdata/fuzz/FuzzParseRBW1Reply/seed-err-nonascii b/testdata/fuzz/FuzzParseRBW1Reply/seed-err-nonascii new file mode 100644 index 0000000..968a005 --- /dev/null +++ b/testdata/fuzz/FuzzParseRBW1Reply/seed-err-nonascii @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("err caf\xc3\xa9 \x00") diff --git a/testdata/fuzz/FuzzParseRBW1Reply/seed-load-ok b/testdata/fuzz/FuzzParseRBW1Reply/seed-load-ok new file mode 100644 index 0000000..7c47b88 --- /dev/null +++ b/testdata/fuzz/FuzzParseRBW1Reply/seed-load-ok @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("ok e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") diff --git a/testdata/fuzz/FuzzParseRBW1Reply/seed-ns-overflow b/testdata/fuzz/FuzzParseRBW1Reply/seed-ns-overflow new file mode 100644 index 0000000..ee27ff1 --- /dev/null +++ b/testdata/fuzz/FuzzParseRBW1Reply/seed-ns-overflow @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("ns 1e999") diff --git a/testdata/fuzz/FuzzParseRBW1Reply/seed-versions b/testdata/fuzz/FuzzParseRBW1Reply/seed-versions new file mode 100644 index 0000000..2922238 --- /dev/null +++ b/testdata/fuzz/FuzzParseRBW1Reply/seed-versions @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("v julia=1.13.3 llvm=20.1.2 threads=1 gc_threads=1 blas_threads=1 Bessels=0.2.8@7a5b9ad5d4c3e8f1a0b2c4d6e8f0a1b3c5d7e9f1") diff --git a/testdata/fuzz/FuzzReadReport/seed-schema2 b/testdata/fuzz/FuzzReadReport/seed-schema2 new file mode 100644 index 0000000..a728653 --- /dev/null +++ b/testdata/fuzz/FuzzReadReport/seed-schema2 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("{\"schema\":2,\"status\":\"measured\",\"max_load\":8,\"rows\":[{\"name\":\"a\",\"func\":\"F\",\"allocs\":0,\"refs\":[{\"id\":\"julia:x\",\"license\":\"MIT\",\"summary\":{\"median\":0,\"q1\":0,\"q3\":0},\"accuracy\":{\"max_ulp\":\"inf\",\"max_ulp_fin\":1.5,\"coverage\":0.5,\"points\":2}}],\"go_accuracy\":{\"max_ulp\":0,\"max_ulp_fin\":0,\"coverage\":1,\"points\":2},\"rounds\":[{\"load_before\":1,\"load_after\":1,\"go_ns\":2,\"ref_ns\":1,\"refs_ns\":{\"julia:x\":1},\"order\":[\"julia:x\",\"go\",\"primary\"]}]}]}") diff --git a/testdata/fuzz/FuzzReadReport/seed-schema2-bad-ulp b/testdata/fuzz/FuzzReadReport/seed-schema2-bad-ulp new file mode 100644 index 0000000..3fb25ad --- /dev/null +++ b/testdata/fuzz/FuzzReadReport/seed-schema2-bad-ulp @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("{\"schema\":2,\"status\":\"deferred\",\"max_load\":8,\"rows\":[{\"name\":\"a\",\"func\":\"F\",\"allocs\":0,\"refs\":[{\"id\":\"x:y\",\"license\":\"MIT\",\"summary\":{\"median\":0,\"q1\":0,\"q3\":0},\"accuracy\":{\"max_ulp\":\"nan\",\"max_ulp_fin\":-1,\"coverage\":9,\"points\":-1}}]}]}") diff --git a/workers.go b/workers.go new file mode 100644 index 0000000..4a28fb9 --- /dev/null +++ b/workers.go @@ -0,0 +1,118 @@ +package refbench + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// JuliaWorkerSource is the Julia program of the rbw1 worker. It reads a kernels +// file (a Dict of scalar functions) named by its first argument. +// +//go:embed workers/julia/worker.jl +var JuliaWorkerSource string + +// CompiledHarnessSource is the C++ harness of the compiled rbw1 workers (C, +// C++, Fortran, Boost.Math): link it with a file defining rbw1_kernels. +// +//go:embed workers/compiled/rbw1.cpp +var CompiledHarnessSource string + +// JuliaConfig describes a Julia reference worker. +type JuliaConfig struct { + // Pkgx is the pkgx executable; Version the pinned Julia, "1.13.3": the + // julialang.org binary through pkgx, never brew. + Pkgx, Version string + // Project is the directory with the pinned Project.toml and Manifest.toml + // (exact versions and git-tree-sha1 content hashes). + Project string + // Kernels is the kernels file (a Dict KERNELS of scalar functions). + Kernels string + // Depot is JULIA_DEPOT_PATH: a directory of the tool (a user cache), where + // `refbench prepare` instantiated and precompiled the project. The worker + // runs with JULIA_PKG_OFFLINE=true: nothing is downloaded during a run. + Depot string + // Dir is the run directory, Funcs the kernels the cases may ask for. + Dir string + Funcs []string +} + +// JuliaArgs returns the arguments of the pkgx command that runs the worker: +// +// pkgx +julialang.org@= -- julia --startup-file=no --history-file=no -t 1 --project= -e +func JuliaArgs(c JuliaConfig) []string { + return []string{"+julialang.org@=" + c.Version, "--", "julia", + "--startup-file=no", "--history-file=no", "-t", "1", "--project=" + c.Project, + "-e", JuliaWorkerSource, c.Kernels} +} + +// JuliaEnv returns the environment of a Julia worker: SingleThreadEnv plus the +// Julia thread counts, the depot and the offline package mode. What the worker +// observed is checked after the start; these variables are not trusted. +func JuliaEnv(c JuliaConfig) []string { + return append(SingleThreadEnv(), + "JULIA_NUM_THREADS=1", "JULIA_NUM_GC_THREADS=1", "JULIA_DEPOT_PATH="+c.Depot, "JULIA_PKG_OFFLINE=true") +} + +// StartJulia starts the Julia worker, asks for its versions and refuses it +// unless it runs the pinned Julia on one thread. The versions (Julia, LLVM, +// every package of the Manifest with its git-tree-sha1) are returned under the +// namespace "julia/" for Options.RefVersions. +func StartJulia(ctx context.Context, c JuliaConfig) (*RBW1, map[string]string, error) { + w, err := StartRBW1(ctx, RBW1Config{Dir: c.Dir, Funcs: c.Funcs}, JuliaEnv(c), c.Pkgx, JuliaArgs(c)...) + if err != nil { + return nil, nil, err + } + v, err := w.Versions() + if err == nil { + err = CheckSingleThread(v) + } + if err == nil && v["julia"] != c.Version { + err = fmt.Errorf("rbw1: the worker runs Julia %q, the pin is %q", v["julia"], c.Version) + } + if err != nil { + _ = w.Close() + return nil, nil, err + } + return w, NamespaceVersions("julia", v), nil +} + +// ReleaseFlags are the default flags of a compiled reference: the library's +// normal release build (-O2, the compiler's own FP contraction, no +// -march=native), not the parity build of NoFMAFlags. The flags are recorded +// and a different flag set is a different Ref.ID. +var ReleaseFlags = []string{"-O2"} + +// BuildWorker compiles a compiled rbw1 worker: cc flags... -o out sources... +// (the harness, a kernel table and the kernels, as sources or objects; a +// source that starts with "-", such as -lgfortran, is passed through after the +// others). It returns what the record needs: the SHA-256 of every source under +// "src/" and the flags under "flags". +func BuildWorker(ctx context.Context, cc, out string, flags, sources []string) (map[string]string, error) { + if len(sources) == 0 { + return nil, errors.New("build worker: no sources") + } + rec := map[string]string{"flags": strings.Join(flags, " ")} + for _, s := range sources { + if strings.HasPrefix(s, "-") { // a library (-lgfortran): after the objects, so that it links + rec["flags"] += " " + s + continue + } + b, err := os.ReadFile(s) + if err != nil { + return nil, err + } + sum := sha256.Sum256(b) + rec["src/"+filepath.Base(s)] = "sha256:" + hex.EncodeToString(sum[:]) + } + if err := BuildC(ctx, cc, out, flags, sources); err != nil { + return nil, err + } + return rec, nil +} diff --git a/workers/compiled/examples/cpp/kernels.cpp b/workers/compiled/examples/cpp/kernels.cpp new file mode 100644 index 0000000..4ac6c71 --- /dev/null +++ b/workers/compiled/examples/cpp/kernels.cpp @@ -0,0 +1,22 @@ +// Example kernel table for the rbw1 harness (C++). Replace with Boost.Math or +// any C/C++ library: one entry per function the cases may ask for. +#include +#include + +extern "C" { +typedef void (*rbw1_fn)(int64_t n, const double *const *cols, double *out); +struct rbw1_kernel { + const char *name; + rbw1_fn fn; +}; + +static void sqrt_k(int64_t n, const double *const *cols, double *out) { + for (int64_t i = 0; i < n; i++) out[i] = std::sqrt(cols[0][i]); +} + +static void add_k(int64_t n, const double *const *cols, double *out) { + for (int64_t i = 0; i < n; i++) out[i] = cols[0][i] + cols[1][i]; +} + +extern const rbw1_kernel rbw1_kernels[] = {{"sqrt", sqrt_k}, {"add", add_k}, {nullptr, nullptr}}; +} diff --git a/workers/compiled/examples/fortran/sqrt.f90 b/workers/compiled/examples/fortran/sqrt.f90 new file mode 100644 index 0000000..de91575 --- /dev/null +++ b/workers/compiled/examples/fortran/sqrt.f90 @@ -0,0 +1,15 @@ +! Example Fortran kernel for the rbw1 harness: AMOS, QUADPACK or MINPACK +! routines are called from subroutines of this shape. +subroutine sqrt_f(n, cols, out) bind(C, name="sqrt_f") + use, intrinsic :: iso_c_binding + implicit none + integer(c_int64_t), value :: n + type(c_ptr), intent(in) :: cols(*) + real(c_double), intent(out) :: out(*) + real(c_double), pointer :: x(:) + integer(c_int64_t) :: i + call c_f_pointer(cols(1), x, [n]) + do i = 1, n + out(i) = sqrt(x(i)) + end do +end subroutine sqrt_f diff --git a/workers/compiled/examples/fortran/table.cpp b/workers/compiled/examples/fortran/table.cpp new file mode 100644 index 0000000..618c651 --- /dev/null +++ b/workers/compiled/examples/fortran/table.cpp @@ -0,0 +1,15 @@ +// Kernel table of the Fortran example: the entries are Fortran subroutines +// declared bind(C) (see sqrt.f90). +#include + +extern "C" { +typedef void (*rbw1_fn)(int64_t n, const double *const *cols, double *out); +struct rbw1_kernel { + const char *name; + rbw1_fn fn; +}; + +void sqrt_f(int64_t n, const double *const *cols, double *out); + +extern const rbw1_kernel rbw1_kernels[] = {{"sqrt", sqrt_f}, {nullptr, nullptr}}; +} diff --git a/workers/compiled/rbw1.cpp b/workers/compiled/rbw1.cpp new file mode 100644 index 0000000..faf412c --- /dev/null +++ b/workers/compiled/rbw1.cpp @@ -0,0 +1,265 @@ +// rbw1 worker harness for compiled references (C, C++, Fortran, Boost.Math). +// See docs/design/multiref.md, section 3.3. Build with refbench.BuildWorker: +// +// c++ -O2 -std=c++17 -o worker rbw1.cpp kernels.cpp [kernels.f90 ...] +// +// The kernels are the worker's fixed function table: a program supplies +// +// extern "C" const rbw1_kernel rbw1_kernels[] = { {"sqrt", sqrt_k}, {nullptr, nullptr} }; +// +// where a kernel evaluates n points, column j of the arguments being cols[j][0..n): +// +// typedef void (*rbw1_fn)(int64_t n, const double *const *cols, double *out); +// +// A Fortran kernel is `subroutine k(n, cols, out) bind(C, name="k")` with +// n of kind c_int64_t, value, and cols a type(c_ptr) array (see +// examples/fortran). Nothing read from stdin is evaluated: a request names an +// entry of rbw1_kernels. Paths must lie inside $RBW1_DIR. No threads are +// started; the checksum of every call is kept so that no loop can be removed. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +typedef void (*rbw1_fn)(int64_t n, const double *const *cols, double *out); +struct rbw1_kernel { + const char *name; + rbw1_fn fn; +}; +extern const rbw1_kernel rbw1_kernels[]; +} + +namespace { + +const size_t kMaxLine = 4096; +const int64_t kMaxData = 256LL << 20; + +// SHA-256 (FIPS 180-4). +struct Sha256 { + uint32_t h[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + uint8_t buf[64]; + uint64_t len = 0; + size_t n = 0; + static uint32_t rotr(uint32_t x, int k) { return (x >> k) | (x << (32 - k)); } + void block(const uint8_t *p) { + static const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + uint32_t w[64]; + for (int i = 0; i < 16; i++) + w[i] = (uint32_t)p[4 * i] << 24 | (uint32_t)p[4 * i + 1] << 16 | (uint32_t)p[4 * i + 2] << 8 | p[4 * i + 3]; + for (int i = 16; i < 64; i++) { + uint32_t s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int i = 0; i < 64; i++) { + uint32_t S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25), ch = (e & f) ^ (~e & g); + uint32_t t1 = hh + S1 + ch + K[i] + w[i]; + uint32_t S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22), mj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + mj; + hh = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2; + } + h[0] += a; h[1] += b; h[2] += c; h[3] += d; h[4] += e; h[5] += f; h[6] += g; h[7] += hh; + } + void update(const uint8_t *p, size_t m) { + len += m; + while (m > 0) { + size_t k = 64 - n < m ? 64 - n : m; + memcpy(buf + n, p, k); + n += k; p += k; m -= k; + if (n == 64) { block(buf); n = 0; } + } + } + std::string hex() { + uint64_t bits = len * 8; + uint8_t pad = 0x80; + update(&pad, 1); + pad = 0; + while (n != 56) update(&pad, 1); + uint8_t l[8]; + for (int i = 0; i < 8; i++) l[i] = (uint8_t)(bits >> (56 - 8 * i)); + update(l, 8); + char out[65]; + for (int i = 0; i < 8; i++) snprintf(out + 8 * i, 9, "%08x", h[i]); + return std::string(out, 64); + } +}; + +struct Data { + std::vector v; + std::vector cols; + std::vector out; + int64_t rows = 0; +}; + +std::map g_data; +std::string g_dir; +volatile double g_sink = 0; + +void reply(const std::string &s) { + fputs(s.c_str(), stdout); + fputc('\n', stdout); + fflush(stdout); +} + +void fail(const std::string &m) { + std::string s = "err "; + for (char c : m.substr(0, 200)) s += (c >= 0x20 && c <= 0x7e) ? c : '?'; + reply(s); +} + +bool split(const std::string &line, std::vector &t) { + size_t i = 0; + while (true) { + size_t j = line.find(' ', i); + std::string tok = line.substr(i, j == std::string::npos ? std::string::npos : j - i); + if (tok.empty()) return false; + t.push_back(tok); + if (j == std::string::npos) return true; + i = j + 1; + } +} + +bool inside(const std::string &p) { + return p.size() > g_dir.size() + 1 && p.compare(0, g_dir.size() + 1, g_dir + "/") == 0 && p.find("..") == std::string::npos; +} + +bool parseInt(const std::string &s, int64_t &v) { + char *end = nullptr; + if (s.empty() || s.size() > 18) return false; + v = strtoll(s.c_str(), &end, 10); + return *end == 0; +} + +rbw1_fn find(const std::string &name) { + for (const rbw1_kernel *k = rbw1_kernels; k->name; k++) + if (name == k->name) return k->fn; + return nullptr; +} + +void call(rbw1_fn f, Data &d) { + f(d.rows, d.cols.data(), d.out.data()); + g_sink = g_sink + d.out[0]; +} + +void handle(const std::string &line) { + std::vector t; + if (line.size() > kMaxLine || !split(line, t)) return fail("malformed line"); + const std::string &c = t[0]; + if (c == "L" && t.size() == 5) { + int64_t cols, rows; + if (!parseInt(t[3], cols) || !parseInt(t[4], rows) || cols <= 0 || rows <= 0 || cols > (1 << 28) || rows > (1 << 28) || + cols * rows * 8 > kMaxData) + return fail("size out of range"); + if (!inside(t[2])) return fail("path outside the run directory"); + FILE *f = fopen(t[2].c_str(), "rb"); + if (!f) return fail("cannot open the file"); + Data d; + d.v.resize((size_t)(cols * rows)); + size_t got = fread(d.v.data(), 8, d.v.size(), f); + int extra = fgetc(f); + fclose(f); + if (got != d.v.size() || extra != EOF) return fail("file size differs from cols*rows*8"); + const uint16_t one = 1; + if (*(const uint8_t *)&one != 1) return fail("big-endian host"); + Sha256 s; + s.update((const uint8_t *)d.v.data(), d.v.size() * 8); + d.rows = rows; + for (int64_t j = 0; j < cols; j++) d.cols.push_back(d.v.data() + j * rows); + d.out.assign((size_t)rows, 0.0); + g_data[t[1]] = std::move(d); + return reply("ok " + s.hex()); + } + if ((c == "W" && t.size() == 3) || (c == "T" && t.size() == 4) || (c == "E" && t.size() == 4)) { + auto it = g_data.find(t[1]); + if (it == g_data.end()) return fail("key not loaded"); + rbw1_fn f = find(t[2]); + if (!f) return fail("function not in the table"); + Data &d = it->second; + if (c == "W") { + call(f, d); + return reply("ok"); + } + if (c == "T") { + int64_t iters; + if (!parseInt(t[3], iters) || iters <= 0) return fail("iters must be positive"); + auto t0 = std::chrono::steady_clock::now(); + for (int64_t i = 0; i < iters; i++) call(f, d); + auto t1 = std::chrono::steady_clock::now(); + char b[64]; + snprintf(b, sizeof b, "ns %.1f", (double)std::chrono::duration_cast(t1 - t0).count()); + return reply(b); + } + if (!inside(t[3])) return fail("path outside the run directory"); + call(f, d); + FILE *o = fopen(t[3].c_str(), "wb"); + if (!o) return fail("cannot create the file"); + size_t w = fwrite(d.out.data(), 8, d.out.size(), o); + int cl = fclose(o); + if (w != d.out.size() || cl != 0) return fail("cannot write the file"); + long nonfinite = 0; + for (double x : d.out) nonfinite += !std::isfinite(x); + return reply("ok " + std::to_string(nonfinite) + " 0"); + } + if (c == "V" && t.size() == 1) { +#ifdef __VERSION__ + std::string ver = __VERSION__; +#else + std::string ver = "unknown"; +#endif + for (char &ch : ver) + if (ch == ' ') ch = '_'; + std::string s = "v compiler=" + ver + " threads=1 ldbl_mant_dig=" + std::to_string(LDBL_MANT_DIG); +#ifdef RBW1_FLAGS + std::string fl = RBW1_FLAGS; + for (char &ch : fl) + if (ch == ' ') ch = '_'; + s += " flags=" + fl; +#endif +#ifdef BOOST_VERSION + s += " boost=" + std::to_string(BOOST_VERSION); +#endif + return reply(s); + } + fail("unknown request"); +} + +} // namespace + +int main() { + const char *dir = getenv("RBW1_DIR"); + if (!dir || !*dir) { + fputs("err RBW1_DIR is not set\n", stdout); + return 1; + } + g_dir = dir; + std::string line; + int ch; + while (true) { + line.clear(); + bool over = false; + while ((ch = getchar()) != EOF && ch != '\n') { + if (line.size() <= kMaxLine) line += (char)ch; + else over = true; + } + if (ch == EOF && line.empty()) break; + if (over) fail("line longer than 4096 bytes"); + else handle(line); + } + return 0; +} diff --git a/workers/julia/worker.jl b/workers/julia/worker.jl new file mode 100644 index 0000000..eb49cd7 --- /dev/null +++ b/workers/julia/worker.jl @@ -0,0 +1,146 @@ +# rbw1 worker for Julia (refbench). See docs/design/multiref.md, section 3.3. +# +# julia --startup-file=no --history-file=no -t 1 --project= -e +# +# kernels.jl is trusted, local, committed code. It defines +# +# KERNELS = Dict{String,Function}("gamma" => x -> SpecialFunctions.gamma(x), ...) +# +# one SCALAR function per name (any arity, Float64 arguments, Float64 result). +# The timed body is `out .= f.(args...)` inside a function barrier on typed +# Vector{Float64}s, with no @fastmath and the default math mode. Nothing read from +# stdin is evaluated: a request names a key of KERNELS, and that is all. +# +# Environment: RBW1_DIR, the run directory; every path of a request must lie +# inside it. +using SHA, TOML, LinearAlgebra + +const MAXLINE = 4096 +const MAXDATA = 256 << 20 +const DIR = get(ENV, "RBW1_DIR", "") + +isempty(DIR) && (println("err RBW1_DIR is not set"); exit(1)) +isempty(ARGS) && (println("err no kernels file"); exit(1)) + +include(abspath(ARGS[1])) # defines KERNELS +@isdefined(KERNELS) || (println("err the kernels file defines no KERNELS"); exit(1)) + +BLAS.set_num_threads(1) + +const DATA = Dict{String,Vector{Vector{Float64}}}() +const ROWS = Dict{String,Int}() +const OUT = Dict{String,Vector{Float64}}() + +oneline(s) = replace(first(string(s), 200), r"[^\x20-\x7e]" => "?") + +function inside(path::AbstractString) + (isabspath(path) && normpath(path) == path && !occursin("..", path) && + startswith(path, DIR * "/")) || error("path outside the run directory") + path +end + +function load(key, path, cols, rows) + cols = parse(Int, cols); rows = parse(Int, rows) + (cols > 0 && rows > 0 && cols * rows * 8 <= MAXDATA) || error("size out of range") + inside(path) + bytes = read(path) + length(bytes) == cols * rows * 8 || error("file holds $(length(bytes)) bytes, want $(cols * rows * 8)") + all = reinterpret(Float64, bytes) # little-endian hosts only; checked below + ENDIAN_BOM == 0x04030201 || error("big-endian host") + DATA[key] = [Vector{Float64}(all[(j - 1) * rows + 1:j * rows]) for j in 1:cols] + ROWS[key] = rows + OUT[key] = Vector{Float64}(undef, rows) + "ok " * bytes2hex(sha256(bytes)) +end + +function kernel(key, fn) + haskey(DATA, key) || error("key not loaded") + haskey(KERNELS, fn) || error("function not in the table") + KERNELS[fn] +end + +# The function barrier: f and the argument tuple are concrete here. +function run!(f, out::Vector{Float64}, args::Tuple, iters::Int) + for _ in 1:iters + out .= f.(args...) + end + return nothing +end + +function evalpoints!(f, out::Vector{Float64}, args::Tuple) + raised = 0 + for i in eachindex(out) + out[i] = try + Float64(f(map(a -> a[i], args)...)) + catch + raised += 1 + NaN + end + end + raised +end + +function versions() + v = Pair{String,String}[] + push!(v, "julia" => string(VERSION)) + push!(v, "llvm" => string(Base.libllvm_version)) + push!(v, "cpu" => Sys.CPU_NAME) + push!(v, "threads" => string(Threads.nthreads())) + push!(v, "gc_threads" => string(Threads.ngcthreads())) + push!(v, "blas_threads" => string(BLAS.get_num_threads())) + # Every package of the active Manifest.toml: version and git-tree-sha1. + manifest = joinpath(dirname(Base.active_project()), "Manifest.toml") + if isfile(manifest) + deps = get(TOML.parsefile(manifest), "deps", Dict()) + for name in sort!(collect(keys(deps))) + e = deps[name][1] + ver = string(get(e, "version", "stdlib")) + tree = get(e, "git-tree-sha1", "") + push!(v, name => isempty(tree) ? ver : ver * "@" * tree) + end + end + "v " * join((k * "=" * replace(val, " " => "_") for (k, val) in v), " ") +end + +function handle(line::AbstractString) + length(line) <= MAXLINE || error("line longer than $MAXLINE") + t = split(line, ' ') + any(isempty, t) && error("empty token") + cmd = t[1] + if cmd == "L" && length(t) == 5 + load(t[2], t[3], t[4], t[5]) + elseif cmd == "W" && length(t) == 3 + f = kernel(t[2], t[3]) + run!(f, OUT[t[2]], Tuple(DATA[t[2]]), 1) + "ok" + elseif cmd == "T" && length(t) == 4 + f = kernel(t[2], t[3]) + iters = parse(Int, t[4]); iters > 0 || error("iters must be positive") + out, args = OUT[t[2]], Tuple(DATA[t[2]]) + t0 = time_ns() + run!(f, out, args, iters) + t1 = time_ns() + "ns " * string(Float64(t1 - t0)) + elseif cmd == "E" && length(t) == 4 + f = kernel(t[2], t[3]) + path = inside(t[4]) + out = Vector{Float64}(undef, ROWS[t[2]]) + raised = evalpoints!(f, out, Tuple(DATA[t[2]])) + write(path, out) + "ok $(count(!isfinite, out)) $raised" + elseif cmd == "V" && length(t) == 1 + versions() + else + error("unknown request") + end +end + +for line in eachline(stdin) + reply = try + handle(line) + catch e + "err " * oneline(e isa ErrorException ? e.msg : sprint(showerror, e)) + end + println(stdout, reply) + flush(stdout) +end From 020fc4abe4271d2ee8845068b4661878d5745cbb Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 15:21:05 +0100 Subject: [PATCH 03/12] README, pkgx syntax, design note README section on several references per case (rounds, verdicts, accuracy, schema 2, rbw1, Julia, compiled workers) and what step 1 leaves out. pkgx 2.11 rejects the "@=" exact form: the worker is started with +julialang.org@ and checked against what it reports. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- README.md | 118 ++++++++++++++++++++++++++++++++++++++++ docs/design/multiref.md | 3 + rbw1_test.go | 2 +- workers.go | 7 ++- 4 files changed, 127 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9b87a99..159158c 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,115 @@ committed sources (`-O2 -ffp-contract=off`), so the library never needs cgo. A repository that already has a cgo module may keep it and time it through the same runner (go-geometry/predicates `bench/` does). +## Several references per case (schema 2) + +SciPy is not the only bar: Julia, Boost.Math and the Fortran libraries are +references too. `Case.Ref` stays the primary reference (existing tables and +sentences stay true); `Case.Refs` adds more, timed in the same rounds +([design](docs/design/multiref.md); the maintainer decisions of +[#15](https://github.com/go-scikit/refbench/issues/15) override it): + +```go +c.Refs = []refbench.Ref{ + {ID: "julia:Bessels.besselk", License: "MIT", Timer: julia.Timer("kv", "bessels_kv"), Eval: julia.Evaler("kv", "bessels_kv", out, nil)}, + {ID: "boost:cyl_bessel_k", License: "BSL-1.0", Timer: boost.Timer("kv", "kv")}, +} +c.RefEval = scipyEval // the primary reference's values on the accuracy points +c.Accuracy = &refbench.Accuracy{Go: goEval, Score: ulpAgainstMpmath, Budget: 4} +``` + +- **Rounds rotate.** With *k* sides (Go, the primary, the extra ones) round *r* + runs them rotated by *r* mod *k*, so every side takes every position once per + *k* rounds, and `Rounds` is rounded up to a multiple of *k*. With two sides it + is exactly the alternation above. A round above the gate is discarded for all + sides. +- **One verdict per reference**, the rule above, plus a headline against the + **fastest side** (`best`, the primary included). **A gap against any faster + reference is a defect, even when that reference is less accurate**: `refbench + issues` drafts one issue per case when Go is slower than any side, titled after + the fastest of them, with every reference, its license, version, ratio and max + ulp in the body. +- **Accuracy in the same row.** `Accuracy.Score` is the repository's own scorer + (special: ulp against the mpmath `hi+lo` fixtures); Go, the primary + (`Case.RefEval`) and every `Ref.Eval` are scored on the same points, once, + ungated, before the rounds. `AccResult` carries `MaxULP` (+Inf, written + `"inf"`, when a finite value was expected and NaN came back), `MaxULPFin` and + `Coverage`, so a reference that refuses half the domain does not look + accurate. Labels: `ahead`, `faster-less-accurate` (an accuracy defect), + `behind`, `tradeoff` (slower than a reference that is less accurate than Go's + stated `Budget`: still a perf defect). The first multi-reference run must + reproduce the fixtures' SciPy column bit for bit (`FirstDiff`), which proves the + inputs reach the workers unaltered. +- **Schema.** A repository with no `Refs` keeps writing schema 1, byte for byte + (a golden test pins it). One with `Refs` writes **schema 2**, which an older + refbench refuses (`perf.json: schema 2, this build reads 1`): bump refbench in + the same PR. `Round` carries a map and a slice, so it is no longer comparable + with `==` (use `reflect.DeepEqual`). The table gains a second section, + "Against other references". +- **CI stays cheap.** `rbtest` starts no runtime: besides allocations, table and + coverage it checks that every `Ref.ID` (`:`, unique) has an entry in + its row, that every reference has a license, that none is GPL unless its kind + is a black-box kind (`r`, `gsl`), and that the table regenerates from the JSON. +- **Calibrate** discards one untimed first call: a reference that compiles on its + first call (Julia) would otherwise last longer than `MinTime` at `iters = 1` + and settle on single-call samples. + +### The rbw1 worker protocol + +One request line, one reply line, plain ASCII, at most 4096 bytes (a longer line +is refused, never buffered), for Julia, R, C++ and Fortran (Python keeps JSON): + +``` +L load float64 little-endian column-major -> ok +W one untimed call (JIT warm-up) -> ok +T time iters calls inside the worker -> ns +E evaluate once, write rows float64 -> ok +V versions and observed threading -> v k=v k=v ... +``` + +`` is a name in a fixed table in the worker (`KERNELS` in Julia, +`rbw1_kernels` in the harness): nothing read from the pipe is evaluated. Paths +must lie in the run directory (`DataDir`: `os.UserCacheDir()/refbench//`, +never `/tmp`), sizes are bounded (256 MiB). `WriteF64` writes an argument file and +returns its SHA-256, `RBW1.Load` compares it with the worker's and refuses a +mismatch naming both digests before anything is timed. `RBW1.Timer` runs one +untimed `W` before the first timed call. `CheckSingleThread` refuses a worker +whose `V` reply does not show `threads=1` (and every `*_threads` at 1): the +environment variables are asked for, what the worker observed is checked. +`ParseRBW1Reply` has a fuzz target and a committed corpus. + +### Julia references + +`refbench.StartJulia(ctx, refbench.JuliaConfig{...})` runs the julialang.org +binary through **pkgx** (never brew), pinned (`JuliaConfig.Version`, refused +unless the worker reports exactly it): + +``` +pkgx +julialang.org@1.13.3 -- julia --startup-file=no --history-file=no -t 1 --project= -e +``` + +with `JULIA_NUM_THREADS=1`, `JULIA_NUM_GC_THREADS=1`, `JULIA_PKG_OFFLINE=true` and +`JULIA_DEPOT_PATH` set. The project (`Project.toml`, `Manifest.toml` with exact +versions and git-tree-sha1 content hashes) is committed in the consumer +repository; the depot is instantiated and precompiled once, outside any gated +run. `kernels.jl` defines `KERNELS = Dict("name" => scalar_function, ...)`; the +timed body is `out .= f.(args...)` in a function barrier on typed +`Vector{Float64}`s, with no `@fastmath`. The versions (Julia, LLVM, every +package of the Manifest with its tree hash) come back under `julia/...` keys for +`Options.RefVersions`, so `-perf.resume` refuses a different set. + +### Compiled references (C, C++, Fortran, Boost.Math) + +`workers/compiled/rbw1.cpp` (embedded as `CompiledHarnessSource`) is the harness +of a small stdin/stdout worker; a program adds a kernel table +(`rbw1_kernels`) and the kernels, in C++ or in Fortran (`bind(C)`); see +`workers/compiled/examples`. `BuildWorker(ctx, cc, out, refbench.ReleaseFlags, +sources)` compiles it (`-O2`, the compiler's own FP contraction, no +`-march=native`: the reference's normal release build; a different flag set is a +different `Ref.ID`) and returns the SHA-256 of every source and the flags for +`ref_versions`. R is not wired yet; when it comes it is `Rscript` only, GPL code is +never linked or read. + ## CI mode: allocations, table, denominator No Python, no timing, on every pull request. `rbtest` is the `go test` glue, @@ -148,6 +257,8 @@ module) declares the parent of the running binary's module path, or the module itself: a binary of one repository run from another's `bench/` directory refuses, naming both module paths. `refbench.RunCI` is what the default mode calls: +0. with `Case.Refs`: every reference has an entry in its row, a license, and no + GPL outside a black-box kind (see above); 1. every case's allocations per call must equal `docs/perf.json`'s (an increase is a regression; a decrease must be recorded); 2. `docs/perf-table.md` must be what `docs/perf.json` generates (no hand edits); @@ -173,6 +284,13 @@ and the commit (`.title.txt` and `.md` with `--out`, for `gh issue create --title ... --body-file`). A deferred report, or one with no valid row, drafts nothing; a partial one drafts from its valid rows. +## Not yet (multi-reference, step 1) + +- Restricting a run to some kinds (`-perf.refs`) and `case@refID` in the + deferred list; the Python `L` request (the NumPy `setup` stays). +- The R worker; a shared home for the per-library kernels (Boost, AMOS, + QUADPACK, MINPACK); `refbench prepare`. + ## Not in v0.1.0 - Closing or commenting on an issue whose row is no longer slower. diff --git a/docs/design/multiref.md b/docs/design/multiref.md index 0d03aa8..35be086 100644 --- a/docs/design/multiref.md +++ b/docs/design/multiref.md @@ -23,6 +23,9 @@ skeleton) is implemented in the PR that adds this file; later steps are not. > in the deferred list, section 3.1) and the Python `L` request (section 3.4): > both need a consumer to design against. The first multi-reference run must > still reproduce the fixtures' SciPy column bit for bit (`FirstDiff`). +> - **pkgx syntax.** pkgx 2.11 rejects `+julialang.org@=1.13.3` ("invalid +> semver"); the worker is started with `+julialang.org@1.13.3` and +> `StartJulia` refuses it unless it reports exactly the pinned version. > - `Round` is no longer a comparable struct (it carries a map and a slice): > compare rounds with `reflect.DeepEqual`. diff --git a/rbw1_test.go b/rbw1_test.go index 921465f..3f79085 100644 --- a/rbw1_test.go +++ b/rbw1_test.go @@ -484,7 +484,7 @@ func TestBombOversizedLineFromAProcess(t *testing.T) { func TestJuliaArgsAndEnv(t *testing.T) { c := juliaCfg(t) args := JuliaArgs(c) - want := []string{"+julialang.org@=1.13.3", "--", "julia", "--startup-file=no", "--history-file=no", "-t", "1", "--project=/p", "-e"} + want := []string{"+julialang.org@1.13.3", "--", "julia", "--startup-file=no", "--history-file=no", "-t", "1", "--project=/p", "-e"} for i, w := range want { if args[i] != w { t.Errorf("args[%d] = %q, want %q", i, args[i], w) diff --git a/workers.go b/workers.go index 4a28fb9..4fb9cc5 100644 --- a/workers.go +++ b/workers.go @@ -45,9 +45,12 @@ type JuliaConfig struct { // JuliaArgs returns the arguments of the pkgx command that runs the worker: // -// pkgx +julialang.org@= -- julia --startup-file=no --history-file=no -t 1 --project= -e +// (pkgx 2.11 rejects the "@=" exact form of the design with "invalid semver"; +// the version is pinned by what the worker reports, which StartJulia checks.) +// +// pkgx +julialang.org@ -- julia --startup-file=no --history-file=no -t 1 --project= -e func JuliaArgs(c JuliaConfig) []string { - return []string{"+julialang.org@=" + c.Version, "--", "julia", + return []string{"+julialang.org@" + c.Version, "--", "julia", "--startup-file=no", "--history-file=no", "-t", "1", "--project=" + c.Project, "-e", JuliaWorkerSource, c.Kernels} } From 9c3459ee7713647d04138b542e49ffb3d2d03551 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 15:25:31 +0100 Subject: [PATCH 04/12] Tests: no real clock for verdicts, no CRLF on the golden files Windows' clock is too coarse for the spin-based verdict tests, and a CRLF checkout changed the golden bytes. Verdict, label and issue tests now run on a hand-built report; .gitattributes marks testdata/golden as -text. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- .gitattributes | 2 ++ multiref_test.go | 46 ++++++++++++++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..161baee --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# The golden files are compared byte for byte: no CRLF conversion on Windows. +testdata/golden/* -text diff --git a/multiref_test.go b/multiref_test.go index 0e1e252..b01f3e0 100644 --- a/multiref_test.go +++ b/multiref_test.go @@ -83,10 +83,28 @@ func TestRunRotatesSidesAndWritesSchema2(t *testing.T) { } } } - // The best is the fastest side, the verdict against it. - if row.Best != "julia:Bessels.gamma" || row.BestVerdict != Slower { - t.Errorf("best = %q %q", row.Best, row.BestVerdict) - } +} + +// synth is a measured report built by hand: Go 12000 ns, the primary +// primaryNs, Bessels 5000, Boost 20000. A timing test on a real clock is not +// reliable on every CI host (Windows' clock is coarse), so the tests of +// verdicts, labels and issues do not time anything. +func synth(primaryNs float64) *Report { + var log []string + c := multiCase(&log) + row := c.row(0) + row.GoAccuracy = &AccResult{Coverage: 1, Points: 2} + row.PrimaryAccuracy = &AccResult{Coverage: 1, Points: 2} + row.GoBudget = 1 + row.Refs[0].Accuracy = &AccResult{MaxULP: 2, MaxULPFin: 2, Coverage: 1, Points: 2} + j, b := "julia:Bessels.gamma", "boost:tgamma" + for i, o := range [][]string{{SideGo, SidePrimary, j, b}, {SidePrimary, j, b, SideGo}, {j, b, SideGo, SidePrimary}, {b, SideGo, SidePrimary, j}} { + row.Rounds = append(row.Rounds, Round{LoadBefore: 1, LoadAfter: 1, GoFirst: i != 1, GoNs: 12000, RefNs: primaryNs, + RefsNs: map[string]float64{j: 5000, b: 20000}, Order: o}) + } + rep := &Report{Schema: SchemaMultiRef, Status: StatusMeasured, MaxLoad: 8, Rows: []Row{row}} + rep.Finish() + return rep } func TestRoundsRoundedUpToTheNumberOfSides(t *testing.T) { @@ -115,8 +133,12 @@ func TestTwoSidesStillAlternate(t *testing.T) { } func TestPerReferenceVerdictsLabelsAndAccuracy(t *testing.T) { - rep, _ := runMulti(t, 4) + rep := synth(10000) row := rep.Rows[0] + // The best is the fastest side, the verdict against it. + if row.Best != "julia:Bessels.gamma" || row.BestVerdict != Slower || row.BestLabel != LabelTradeoff { + t.Errorf("best = %q %q %q", row.Best, row.BestVerdict, row.BestLabel) + } bessels, boost := row.Refs[0], row.Refs[1] if bessels.Verdict != Slower || bessels.Ratio <= 1 { t.Errorf("bessels = %+v", bessels) @@ -167,9 +189,7 @@ func TestLabel(t *testing.T) { } func TestIssueAgainstAnyFasterReference(t *testing.T) { - rep, _ := runMulti(t, 4) - // Make the primary slower than Go (Go ties or beats it) and keep the - // less accurate Julia reference faster: the issue is still drafted. + rep := synth(10000) issues, err := Issues(rep) if err != nil || len(issues) != 1 { t.Fatalf("issues = %v, %v", issues, err) @@ -179,13 +199,7 @@ func TestIssueAgainstAnyFasterReference(t *testing.T) { t.Errorf("issue = %+v", is) } // Only a reference that is not the primary is faster. - var log []string - c := multiCase(&log) - c.Ref = fixedTimer(15000, &log, SidePrimary) // Go beats SciPy, but not Julia - rep2, err := Run(context.Background(), []Case{c}, Options{Rounds: 4, MinTime: time.Millisecond, Load: func() (float64, error) { return 1, nil }}) - if err != nil { - t.Fatal(err) - } + rep2 := synth(15000) // Go beats the primary, but not Julia if rep2.Rows[0].Verdict != Faster { t.Fatalf("setup: primary verdict %q", rep2.Rows[0].Verdict) } @@ -196,7 +210,7 @@ func TestIssueAgainstAnyFasterReference(t *testing.T) { } func TestSchema2RoundTripAndTable(t *testing.T) { - rep, _ := runMulti(t, 4) + rep := synth(10000) var b bytes.Buffer if err := rep.Write(&b); err != nil { t.Fatal(err) From 0936bbc24454e4012d8a5ba26425320223b721fa Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 15:29:00 +0100 Subject: [PATCH 05/12] Tests: cover StartRBW1's default environment without a compiler The compiled-worker test is skipped on Windows, which left the default SingleThreadEnv branch of StartRBW1 uncovered there. Cover it with the fake process; 100% coverage now holds with that test skipped. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- multiref_more_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/multiref_more_test.go b/multiref_more_test.go index bda28f2..fa7f5a6 100644 --- a/multiref_more_test.go +++ b/multiref_more_test.go @@ -103,6 +103,18 @@ func TestStartRBW1PipeErrors(t *testing.T) { } } +func TestStartRBW1DefaultEnvIsSingleThread(t *testing.T) { + helperRBW1(t, "ok") + w, err := StartRBW1(context.Background(), RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}, nil, "x") + if err != nil { + t.Fatal(err) + } + defer w.Close() + if !strings.Contains(strings.Join(w.cmd.Env, "\n"), "OMP_NUM_THREADS=1") { + t.Error("the default environment is not SingleThreadEnv") + } +} + func TestRunStopsOnATimerErrorInARound(t *testing.T) { var log []string c := multiCase(&log) From 2058210a58e3a9aa3d7b65aadadf781f5b5a01f8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 15:59:58 +0100 Subject: [PATCH 06/12] CI: covblocks names the uncovered blocks instead of grep on the rounded total Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5716eea..79cadea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,8 +36,10 @@ jobs: COVERPKG=$(go list ./... | paste -sd, -) go test -race -coverpkg="$COVERPKG" -coverprofile=cover.out ./... go tool cover -func=cover.out | tail -n 40 - go tool cover -func=cover.out | tail -1 | grep -q '100.0%' || { - echo "::error::coverage below 100%"; exit 1; } + # go tool cover's total is rounded to one decimal (99.95 reads 100.0); + # covblocks lists every block the profile never entered (exit 1 if + # any, exit 2 on an empty or unreadable profile). + go run github.com/go-fleettools/fleettools/cmd/covblocks@v0.104.0 cover.out # int is 32 bits here: v0.1.0 did not compile on 386 and arm. arch-386: From ddef2e2c17d0325d234082066e77cc5bb2eebf85 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 16:04:58 +0100 Subject: [PATCH 07/12] Tests: cover rbtest defaults, use native paths in the rbw1 request tests covblocks (now the CI gate) named rbtest.go:78-88 on every OS and, on Windows, the rbw1 Load/Eval/ask refusals that the mixed-separator test paths never reached. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- multiref_more_test.go | 3 ++- rbtest/defaults_test.go | 26 ++++++++++++++++++++++++++ rbtest/rbtest.go | 5 ++++- rbw1_test.go | 21 ++++++++++++--------- 4 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 rbtest/defaults_test.go diff --git a/multiref_more_test.go b/multiref_more_test.go index fa7f5a6..d362964 100644 --- a/multiref_more_test.go +++ b/multiref_more_test.go @@ -5,6 +5,7 @@ import ( "context" "io" "os/exec" + "path/filepath" "strings" "testing" "time" @@ -76,7 +77,7 @@ func TestEvalerError(t *testing.T) { if err := w.Load("x", path, 1, rows, sha); err != nil { t.Fatal(err) } - if _, err := w.Evaler("x", "f", dir+"/o", nil)(); err == nil { + if _, err := w.Evaler("x", "f", filepath.Join(dir, "o"), nil)(); err == nil { t.Error("worker error lost") } } diff --git a/rbtest/defaults_test.go b/rbtest/defaults_test.go new file mode 100644 index 0000000..cc7680b --- /dev/null +++ b/rbtest/defaults_test.go @@ -0,0 +1,26 @@ +package rbtest + +import ( + "runtime/debug" + "testing" +) + +// The tests replace flagSet and buildModule; these keep the originals, taken +// at package initialisation, so that their bodies run too (covblocks lists +// every block no test entered). +var origFlagSet, origBuildModule = flagSet, buildModule + +func TestDefaultFlagSetAndBuildModule(t *testing.T) { + if origFlagSet("perf.run") { + t.Error("perf.run reported as given on the command line") + } + if got := origBuildModule(); got == "" { + t.Error("the test binary has no module path") + } + old := readBuildInfo + defer func() { readBuildInfo = old }() + readBuildInfo = func() (*debug.BuildInfo, bool) { return nil, false } + if got := origBuildModule(); got != "" { + t.Errorf("module = %q without build info", got) + } +} diff --git a/rbtest/rbtest.go b/rbtest/rbtest.go index 915fca6..7375d0e 100644 --- a/rbtest/rbtest.go +++ b/rbtest/rbtest.go @@ -80,9 +80,12 @@ var flagSet = func(name string) bool { return set } +// readBuildInfo is replaceable by tests. +var readBuildInfo = debug.ReadBuildInfo + // buildModule is the module path of the running binary; tests replace it. var buildModule = func() string { - if bi, ok := debug.ReadBuildInfo(); ok { + if bi, ok := readBuildInfo(); ok { return bi.Main.Path } return "" diff --git a/rbw1_test.go b/rbw1_test.go index 3f79085..67dc23a 100644 --- a/rbw1_test.go +++ b/rbw1_test.go @@ -178,17 +178,17 @@ func TestRBW1RefusesBeforeSending(t *testing.T) { "dotdot": w.Load("x", dir+"/../o.f64", 1, 1, sha), "the dir itself": w.Load("x", dir, 1, 1, sha), "space in path": w.Load("x", dir+"/a b", 1, 1, sha), - "zero cols": w.Load("x", dir+"/a", 0, 1, sha), - "huge": w.Load("x", dir+"/a", 1<<20, 1<<20, sha), - "overflow 32 bits": w.Load("x", dir+"/a", 1<<28, 1<<28, sha), - "bad load key": w.Load("", dir+"/a", 1, 1, sha), + "zero cols": w.Load("x", filepath.Join(dir, "a"), 0, 1, sha), + "huge": w.Load("x", filepath.Join(dir, "a"), 1<<20, 1<<20, sha), + "overflow 32 bits": w.Load("x", filepath.Join(dir, "a"), 1<<28, 1<<28, sha), + "bad load key": w.Load("", filepath.Join(dir, "a"), 1, 1, sha), "eval outside": evalErr(w, "/etc/passwd"), "time bad fn": timeErr(w, "g", 1), "time zero iters": timeErr(w, "f", 0), "eval bad fn": evalFnErr(w, dir), "unloaded evaler": evalerErr(w, dir), "warm in timer": timerErr(w, "g"), - "huge request line": w.Load("x", dir+"/"+strings.Repeat("a", RBW1MaxLine), 1, 1, sha), + "huge request line": w.Load("x", filepath.Join(dir, strings.Repeat("a", RBW1MaxLine)), 1, 1, sha), } { if err == nil { t.Errorf("%s: accepted", name) @@ -204,9 +204,12 @@ func timeErr(w *RBW1, fn string, it int) error { _, err := w.Time("x", fn, it) return err } -func evalFnErr(w *RBW1, dir string) error { _, _, err := w.Eval("x", "zz", dir+"/o"); return err } +func evalFnErr(w *RBW1, dir string) error { + _, _, err := w.Eval("x", "zz", filepath.Join(dir, "o")) + return err +} func evalerErr(w *RBW1, dir string) error { - _, err := w.Evaler("never", "f", dir+"/o", nil)() + _, err := w.Evaler("never", "f", filepath.Join(dir, "o"), nil)() return err } func timerErr(w *RBW1, fn string) error { _, err := w.Timer("x", fn)(1); return err } @@ -224,7 +227,7 @@ func TestRBW1RefusesBadReplies(t *testing.T) { } for _, r := range []string{"ns 1", "ok", "ok zz", "ok " + strings.Repeat("A", 64), "ok " + strings.Repeat("1", 64), "err no"} { reply = r - check("load", w.Load("x", dir+"/a", 1, 1, sha)) + check("load", w.Load("x", filepath.Join(dir, "a"), 1, 1, sha)) } for _, r := range []string{"ns 1", "ok 1", "err no", "v a=1"} { reply = r @@ -237,7 +240,7 @@ func TestRBW1RefusesBadReplies(t *testing.T) { } for _, r := range []string{"ns 1", "ok", "ok 1", "ok -1 0", "ok 0 x", "ok 1 2 3", "err x"} { reply = r - _, _, err := w.Eval("x", "f", dir+"/o") + _, _, err := w.Eval("x", "f", filepath.Join(dir, "o")) check("eval", err) } for _, r := range []string{"ok", "ns 1", "err x"} { From d37a87bc775fa229405a5c4d4ed2230dadaff727 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 16:28:57 +0100 Subject: [PATCH 08/12] Review fixes: one schema predicate, verified keys, Williams rounds, MaxULPFin labels - Schema: one predicate (Row.usesMultiRef) decides what Finish writes and what a schema 1 file may carry; a case with Accuracy and no Refs now round-trips. - A key whose last Load did not verify cannot be warmed, timed or evaluated. - Rounds follow a Williams design: every ordered pair of sides is adjacent equally often (a rotation kept first-order carryover). - Labels are judged on MaxULPFin; special-value disagreements are counted separately (AccResult.Disagree). - Compiled harness reports where its thread count comes from. - worker.jl: bound each factor before the product; a kernel that raises on every point is refused; TestJuliaWorkerReal (REFBENCH_JULIA=1) runs it. - README and design: the SciPy-column check is stated as deferred to step 2. - Tests for the request and 256 MiB boundaries and the Eval out-path. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- README.md | 57 ++++++++--- docs/design/multiref.md | 11 ++- julia_real_test.go | 91 ++++++++++++++++++ multiref.go | 32 +++++-- multiref_test.go | 13 ++- rbw1.go | 7 ++ rbw1_test.go | 31 ++++-- review_test.go | 196 ++++++++++++++++++++++++++++++++++++++ runner.go | 71 ++++++++++++-- table.go | 13 ++- williams_test.go | 57 +++++++++++ workers/compiled/rbw1.cpp | 29 +++++- workers/julia/worker.jl | 10 +- 13 files changed, 571 insertions(+), 47 deletions(-) create mode 100644 julia_real_test.go create mode 100644 review_test.go create mode 100644 williams_test.go diff --git a/README.md b/README.md index 159158c..e183630 100644 --- a/README.md +++ b/README.md @@ -119,10 +119,13 @@ c.RefEval = scipyEval // the primary reference's values on the accuracy points c.Accuracy = &refbench.Accuracy{Go: goEval, Score: ulpAgainstMpmath, Budget: 4} ``` -- **Rounds rotate.** With *k* sides (Go, the primary, the extra ones) round *r* - runs them rotated by *r* mod *k*, so every side takes every position once per - *k* rounds, and `Rounds` is rounded up to a multiple of *k*. With two sides it - is exactly the alternation above. A round above the gate is discarded for all +- **Balanced rounds.** With *k* sides (Go, the primary, the extra ones) the + rounds follow a Williams design (a balanced Latin square, `k` orders for even + *k*, `2k` for odd): every side takes every position equally often **and every + ordered pair of sides is adjacent equally often**, so no side always follows + the same other one (a plain rotation balances positions but not that + carryover). `Rounds` is rounded up to a whole period. With two sides it is + exactly the alternation above. A round above the gate is discarded for all sides. - **One verdict per reference**, the rule above, plus a headline against the **fastest side** (`best`, the primary included). **A gap against any faster @@ -134,15 +137,27 @@ c.Accuracy = &refbench.Accuracy{Go: goEval, Score: ulpAgainstMpmath, Budget: 4} (special: ulp against the mpmath `hi+lo` fixtures); Go, the primary (`Case.RefEval`) and every `Ref.Eval` are scored on the same points, once, ungated, before the rounds. `AccResult` carries `MaxULP` (+Inf, written - `"inf"`, when a finite value was expected and NaN came back), `MaxULPFin` and - `Coverage`, so a reference that refuses half the domain does not look - accurate. Labels: `ahead`, `faster-less-accurate` (an accuracy defect), - `behind`, `tradeoff` (slower than a reference that is less accurate than Go's - stated `Budget`: still a perf defect). The first multi-reference run must - reproduce the fixtures' SciPy column bit for bit (`FirstDiff`), which proves the - inputs reach the workers unaltered. -- **Schema.** A repository with no `Refs` keeps writing schema 1, byte for byte - (a golden test pins it). One with `Refs` writes **schema 2**, which an older + `"inf"`, when a finite value was expected and NaN came back), `MaxULPFin`, + `Disagree` (the number of such special-value disagreements) and `Coverage`, so + a reference that refuses half the domain does not look accurate. Labels: + `ahead`, `faster-less-accurate` (an accuracy defect), `behind`, `tradeoff` + (slower than a reference that is less accurate than Go's stated `Budget`: still + a perf defect). **The labels are judged on `MaxULPFin`**, the worst error over + the finite results: one special-value disagreement must not decide between + `behind` and `tradeoff`; it is reported (`MaxULP`, `Disagree`), not judged. +- **Not enforced yet: the SciPy column check.** Decision of #15: the first + multi-reference run must reproduce the fixtures' SciPy column bit for bit, + which proves the inputs reach the workers unaltered. Step 1 has the helper + (`FirstDiff`, tested on its own) but **no gate calls it**: the Python worker + has no `L` request yet, so nothing can run SciPy on the file the other workers + read. This is deferred to step 2 (Python `L`, then a gate with a planted + negative control). What step 1 does enforce is the SHA-256 of every argument + file, for the workers that have `L`. +- **Schema.** A repository with no `Refs` and no `Accuracy` keeps writing schema 1, + byte for byte (a golden test pins it). One with `Refs`, or with `Accuracy` + (`go_accuracy`, `primary_accuracy`, `go_budget` are schema 2 fields; one + predicate decides what is written and what a schema 1 file may carry), writes + **schema 2**, which an older refbench refuses (`perf.json: schema 2, this build reads 1`): bump refbench in the same PR. `Round` carries a map and a slice, so it is no longer comparable with `==` (use `reflect.DeepEqual`). The table gains a second section, @@ -173,12 +188,26 @@ V versions and observed threading -> v k=v must lie in the run directory (`DataDir`: `os.UserCacheDir()/refbench//`, never `/tmp`), sizes are bounded (256 MiB). `WriteF64` writes an argument file and returns its SHA-256, `RBW1.Load` compares it with the worker's and refuses a -mismatch naming both digests before anything is timed. `RBW1.Timer` runs one +mismatch naming both digests, and **a key whose last `Load` did not verify +(a mismatch, or a reload that failed after an earlier success) cannot be warmed, +timed or evaluated**. `RBW1.Timer` runs one untimed `W` before the first timed call. `CheckSingleThread` refuses a worker whose `V` reply does not show `threads=1` (and every `*_threads` at 1): the environment variables are asked for, what the worker observed is checked. +Julia reports its real thread, GC-thread and BLAS-thread counts. The compiled +harness reports `threads_source`: `omp` (`omp_get_max_threads()`, when built with +OpenMP), `proc` (the process's thread count from `/proc/self/status`, Linux) or +`assumed` (the harness starts no thread; **a linked library that starts its own is +not detected there**: the check is unverified for compiled workers on those +hosts, and the record says so). A Julia kernel that raises on every point is a +broken kernel and is refused, not reported as an all-NaN column. `ParseRBW1Reply` has a fuzz target and a committed corpus. +**Gap:** `worker.jl` needs Julia and runs in no CI lane (CI installs no runtime). +`REFBENCH_JULIA=1 go test -run TestJuliaWorkerReal .` runs it for real (pkgx, +pinned Julia, kernels that need no package) on a host that has them; it passed +on the reference host before this release. + ### Julia references `refbench.StartJulia(ctx, refbench.JuliaConfig{...})` runs the julialang.org diff --git a/docs/design/multiref.md b/docs/design/multiref.md index 35be086..47c1156 100644 --- a/docs/design/multiref.md +++ b/docs/design/multiref.md @@ -21,8 +21,15 @@ skeleton) is implemented in the PR that adds this file; later steps are not. > (Q4), SciPy stays primary (Q8). > - **Step 1 leaves out** the per-kind restriction (`-perf.refs`, `case@refID` > in the deferred list, section 3.1) and the Python `L` request (section 3.4): -> both need a consumer to design against. The first multi-reference run must -> still reproduce the fixtures' SciPy column bit for bit (`FirstDiff`). +> both need a consumer to design against. The decision that the first +> multi-reference run must reproduce the fixtures' SciPy column bit for bit is +> **not enforced in step 1** (`FirstDiff` exists, no gate calls it: the Python +> worker has no `L` yet); it is deferred to step 2 with a planted negative +> control. +> - **Round order.** Section 3.1's plain rotation keeps first-order carryover +> (Go always follows the same side); the rounds follow a Williams design +> instead (`2k` orders for odd `k`). Labels are judged on `MaxULPFin`, not on +> `MaxULP`. > - **pkgx syntax.** pkgx 2.11 rejects `+julialang.org@=1.13.3` ("invalid > semver"); the worker is started with `+julialang.org@1.13.3` and > `StartJulia` refuses it unless it reports exactly the pinned version. diff --git a/julia_real_test.go b/julia_real_test.go new file mode 100644 index 0000000..219ea8d --- /dev/null +++ b/julia_real_test.go @@ -0,0 +1,91 @@ +package refbench + +import ( + "context" + "math" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestJuliaWorkerReal runs workers/julia/worker.jl in the pinned Julia, through +// pkgx, with kernels that need no package. It never runs in CI (no Julia +// there): set REFBENCH_JULIA=1 on a host that has pkgx and network access to +// fetch julialang.org once. +func TestJuliaWorkerReal(t *testing.T) { + if os.Getenv("REFBENCH_JULIA") == "" { + t.Skip("set REFBENCH_JULIA=1 (needs pkgx; runs the real Julia worker)") + } + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + kernels := filepath.Join(dir, "kernels.jl") + src := `const KERNELS = Dict{String,Function}( + "sqrt" => x -> sqrt(x), + "add" => (a, b) -> a + b, + "dom" => x -> x < 0 ? throw(DomainError(x)) : x, +) +` + if err := os.WriteFile(kernels, []byte(src), 0o644); err != nil { + t.Fatal(err) + } + project, depot := filepath.Join(dir, "project"), filepath.Join(dir, "depot") + for _, d := range []string{project, depot} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + run := filepath.Join(dir, "run") + if err := os.MkdirAll(run, 0o755); err != nil { + t.Fatal(err) + } + w, v, err := StartJulia(context.Background(), JuliaConfig{Pkgx: "pkgx", Version: "1.13.3", Project: project, Kernels: kernels, + Depot: depot, Dir: run, Funcs: []string{"sqrt", "add", "dom"}}) + if err != nil { + t.Fatal(err) + } + defer w.Close() + if v["julia/julia"] != "1.13.3" || v["julia/threads"] != "1" { + t.Errorf("versions = %v", v) + } + x := []float64{1, 4, 9, -1} + path, rows, sha, err := WriteF64(run, "x", x) + if err != nil { + t.Fatal(err) + } + if err := w.Load("x", path, 1, rows, sha); err != nil { + t.Fatal(err) + } + path2, _, sha2, err := WriteF64(run, "xy", x, x) + if err != nil { + t.Fatal(err) + } + if err := w.Load("xy", path2, 2, rows, sha2); err != nil { + t.Fatal(err) + } + var nf, raised int + got, err := w.Evaler("x", "dom", filepath.Join(run, "o.f64"), func(a, b int) { nf, raised = a, b })() + if err != nil || nf != 1 || raised != 1 || !math.IsNaN(got[3]) || got[2] != 9 { + t.Errorf("dom: %v %d %d %v", got, nf, raised, err) + } + if got, err = w.Evaler("xy", "add", filepath.Join(run, "a.f64"), nil)(); err != nil || got[1] != 8 { + t.Errorf("add: %v %v", got, err) + } + if _, err := Calibrate(w.Timer("xy", "add"), 10*time.Millisecond); err != nil { + t.Error(err) + } + // A kernel of the wrong arity raises on every point: refused, not an all-NaN column. + if _, err := w.Evaler("xy", "sqrt", filepath.Join(run, "b.f64"), nil)(); err == nil || !strings.Contains(err.Error(), "every point") { + t.Errorf("wrong arity: %v", err) + } + // The worker's own refusals, bypassing the client's: a product that wraps + // in Int64, a path outside the run directory, a function outside its table. + for _, line := range []string{"L y " + path + " 3037000500 3037000500", "L y " + path + " 1 536870913", "L y /etc/passwd 1 1", "T x nofn 1", "W nokey sqrt", "E x sqrt /tmp/o", "zz"} { + if _, err := w.ask(strings.Split(line, " ")...); err == nil { + t.Errorf("the Julia worker accepted %q", line) + } + } +} diff --git a/multiref.go b/multiref.go index d280ed5..a77c673 100644 --- a/multiref.go +++ b/multiref.go @@ -103,12 +103,16 @@ type AccResult struct { Coverage float64 `json:"coverage"` // Points is the number of points scored. Points int `json:"points"` + // Disagree counts the points where the side returned a non-finite value + // where a finite one was expected, or the reverse (the points that make + // MaxULP infinite). Reported, never judged. + Disagree int `json:"disagree,omitempty"` } func (a AccResult) valid() bool { m, f := float64(a.MaxULP), float64(a.MaxULPFin) return !math.IsNaN(m) && m >= 0 && !math.IsNaN(f) && f >= 0 && - !math.IsNaN(a.Coverage) && a.Coverage >= 0 && a.Coverage <= 1 && a.Points >= 0 + !math.IsNaN(a.Coverage) && a.Coverage >= 0 && a.Coverage <= 1 && a.Points >= 0 && a.Disagree >= 0 && a.Disagree <= a.Points } // RefRow is one extra reference's result in a row. ID, Name and License come @@ -144,13 +148,19 @@ const ( // Label combines the time verdict with the accuracy. Without accuracy on both // sides a faster Go has no label (no accuracy claim is made) and a slower Go is // behind. +// +// The judge is MaxULPFin, the worst error over the points where each side +// returned a finite value: one special-value disagreement (MaxULP = +Inf, a +// NaN where a number was expected) must not decide between "behind" and +// "tradeoff". Those disagreements are reported separately (AccResult.Disagree, +// and MaxULP itself). func Label(verdict string, goAcc, refAcc *AccResult, budget float64) string { if verdict == Slower { if goAcc != nil && refAcc != nil { if budget <= 0 { - budget = float64(goAcc.MaxULP) + budget = float64(goAcc.MaxULPFin) } - if float64(refAcc.MaxULP) > budget { + if float64(refAcc.MaxULPFin) > budget { return LabelTradeoff } } @@ -159,7 +169,7 @@ func Label(verdict string, goAcc, refAcc *AccResult, budget float64) string { if verdict == "" || goAcc == nil || refAcc == nil { return "" } - if goAcc.MaxULP <= refAcc.MaxULP { + if goAcc.MaxULPFin <= refAcc.MaxULPFin { return LabelAhead } return LabelFasterLessAccurate @@ -224,10 +234,17 @@ func (r RefRow) label() string { return r.ID } -// usesRefs reports whether any row carries extra references. +// usesMultiRef reports whether a row carries any schema 2 field: extra +// references, or accuracy (Go's, the primary's) and its budget. One predicate +// decides the schema Finish writes and what a schema 1 file may not contain. +func (row Row) usesMultiRef() bool { + return len(row.Refs) > 0 || row.GoAccuracy != nil || row.PrimaryAccuracy != nil || row.GoBudget != 0 +} + +// usesRefs reports whether any row needs schema 2. func (r *Report) usesRefs() bool { for _, row := range r.Rows { - if len(row.Refs) > 0 { + if row.usesMultiRef() { return true } } @@ -348,8 +365,7 @@ func FirstDiff(a, b []float64) int { // checkMultiRef validates the schema 2 fields of a row read from a file. func checkMultiRef(row Row, schema int) error { - extra := len(row.Refs) > 0 || row.GoAccuracy != nil || row.PrimaryAccuracy != nil || row.GoBudget != 0 - if schema == SchemaVersion && extra { + if schema == SchemaVersion && row.usesMultiRef() { return errors.New("a schema 1 file carries extra-reference fields (schema 2 is required)") } if !finiteNonNeg(row.GoBudget) { diff --git a/multiref_test.go b/multiref_test.go index b01f3e0..8268780 100644 --- a/multiref_test.go +++ b/multiref_test.go @@ -160,9 +160,13 @@ func TestPerReferenceVerdictsLabelsAndAccuracy(t *testing.T) { } func TestLabel(t *testing.T) { - two := &AccResult{MaxULP: 2} - one := &AccResult{MaxULP: 1} + two := &AccResult{MaxULP: 2, MaxULPFin: 2} + one := &AccResult{MaxULP: 1, MaxULPFin: 1} zero := &AccResult{} + // One special-value disagreement (MaxULP = +Inf) must not decide: only the + // worst error over the finite results does. + inf1 := &AccResult{MaxULP: ULP(math.Inf(1)), MaxULPFin: 1, Disagree: 1, Points: 5} + inf3 := &AccResult{MaxULP: ULP(math.Inf(1)), MaxULPFin: 3, Disagree: 1, Points: 5} cases := []struct { v string g, r *AccResult @@ -180,6 +184,11 @@ func TestLabel(t *testing.T) { {Faster, two, one, 0, LabelFasterLessAccurate, ""}, {Faster, zero, nil, 0, "", "no ref accuracy"}, {"", one, one, 0, "", "no verdict"}, + {Slower, inf1, inf1, 0, LabelBehind, "both infinite, same finite error: behind (was always behind)"}, + {Slower, inf1, inf3, 0, LabelTradeoff, "Go's own max is infinite: the finite error still decides"}, + {Slower, inf3, inf1, 0, LabelBehind, "ref more accurate on finite points"}, + {Faster, inf1, inf3, 0, LabelAhead, "finite error decides ahead"}, + {Faster, inf3, inf1, 0, LabelFasterLessAccurate, "finite error decides faster-less-accurate"}, } for _, c := range cases { if got := Label(c.v, c.g, c.r, c.budget); got != c.want { diff --git a/rbw1.go b/rbw1.go index d0a1c0d..7a9f047 100644 --- a/rbw1.go +++ b/rbw1.go @@ -262,6 +262,12 @@ func (w *RBW1) check(key, fn string) error { if !w.funcs[fn] { return fmt.Errorf("rbw1: function %q is not in the worker's table", fn) } + // Only a key whose LAST Load verified the digest may be warmed, timed or + // evaluated: a mismatch, or a reload that failed after an earlier success, + // leaves the worker holding data that is not the Go side's. + if _, ok := w.rows[key]; !ok { + return fmt.Errorf("rbw1: key %q has no verified load", key) + } return nil } @@ -283,6 +289,7 @@ func (w *RBW1) Load(key, path string, cols, rows int, want string) error { if !rbw1KeyPattern.MatchString(key) { return fmt.Errorf("rbw1: key %q is not a plain identifier", key) } + delete(w.rows, key) // verified again only if this load succeeds if err := w.inDir(path); err != nil { return err } diff --git a/rbw1_test.go b/rbw1_test.go index 67dc23a..39405c8 100644 --- a/rbw1_test.go +++ b/rbw1_test.go @@ -100,6 +100,7 @@ func fake(t *testing.T, dir string, funcs []string, respond func(line string) st if err != nil { t.Fatal(err) } + w.rows["x"] = 1 // tests that need an unverified key start with a failed Load t.Cleanup(func() { inW.Close(); outR.Close() }) return w } @@ -199,8 +200,13 @@ func TestRBW1RefusesBeforeSending(t *testing.T) { } } -func evalErr(w *RBW1, p string) error { _, _, err := w.Eval("x", "f", p); return err } +func evalErr(w *RBW1, p string) error { + w.rows["x"] = 1 // a verified key: the refusal must come from the path + _, _, err := w.Eval("x", "f", p) + return err +} func timeErr(w *RBW1, fn string, it int) error { + w.rows["x"] = 1 // the map of cases runs in any order, and Load un-verifies the key _, err := w.Time("x", fn, it) return err } @@ -231,15 +237,18 @@ func TestRBW1RefusesBadReplies(t *testing.T) { } for _, r := range []string{"ns 1", "ok 1", "err no", "v a=1"} { reply = r + w.rows["x"] = 1 // verified: the replies are what is under test check("warm", w.Warm("x", "f")) } for _, r := range []string{"ok", "v a=1", "ns 1e300", "err x"} { reply = r + w.rows["x"] = 1 _, err := w.Time("x", "f", 1) check("time", err) } for _, r := range []string{"ns 1", "ok", "ok 1", "ok -1 0", "ok 0 x", "ok 1 2 3", "err x"} { reply = r + w.rows["x"] = 1 _, _, err := w.Eval("x", "f", filepath.Join(dir, "o")) check("eval", err) } @@ -278,6 +287,7 @@ func TestRBW1ProcessEnded(t *testing.T) { if err != nil { t.Fatal(err) } + w.rows["x"] = 1 inW.Close() // the write fails if err := w.Warm("x", "f"); err == nil { t.Error("write to a closed worker accepted") @@ -291,6 +301,7 @@ func TestRBW1ProcessEnded(t *testing.T) { go func() { io.Copy(io.Discard, in2R) }() out2W.Close() w2, _ := NewRBW1(in2W, out2R, RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}) + w2.rows["x"] = 1 if err := w2.Warm("x", "f"); err == nil || !strings.Contains(err.Error(), "ended") { t.Errorf("eof: %v", err) } @@ -298,6 +309,7 @@ func TestRBW1ProcessEnded(t *testing.T) { in3R, in3W := io.Pipe() go func() { io.Copy(io.Discard, in3R) }() w3, _ := NewRBW1(in3W, errReader{}, RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}) + w3.rows["x"] = 1 if err := w3.Warm("x", "f"); err == nil || !strings.Contains(err.Error(), "io") { t.Errorf("read error: %v", err) } @@ -348,6 +360,7 @@ func TestBombOversizedLine(t *testing.T) { if err != nil { t.Fatal(err) } + w.rows["x"] = 1 var before, after runtime.MemStats runtime.ReadMemStats(&before) err = w.Warm("x", "f") @@ -426,6 +439,7 @@ func TestStartJulia(t *testing.T) { t.Fatal(err) } defer w.Close() + w.rows["x"] = 1 if v["julia/julia"] != "1.13.3" || v["julia/Bessels"] != "0.2.8@abc" || v["julia/dir"] != c.Dir { t.Errorf("versions = %v", v) } @@ -470,6 +484,7 @@ func TestBombOversizedLineFromAProcess(t *testing.T) { if err != nil { t.Fatal(err) } + w.rows["x"] = 1 done := make(chan error, 1) go func() { done <- w.Warm("x", "f") }() select { @@ -683,7 +698,7 @@ func exerciseCompiled(t *testing.T, bin, fn string) { } defer w.Close() v, err := w.Versions() - if err != nil || CheckSingleThread(v) != nil || v["compiler"] == "" { + if err != nil || CheckSingleThread(v) != nil || v["compiler"] == "" || v["threads_source"] == "" { t.Fatalf("versions = %v %v", v, err) } x := []float64{1, 4, 9, -1, 16} @@ -691,13 +706,17 @@ func exerciseCompiled(t *testing.T, bin, fn string) { if err != nil { t.Fatal(err) } - if err := w.Load("x", path, 1, rows, sha); err != nil { - t.Fatal(err) - } - // Negative control: a planted wrong digest is refused with both digests. + // Negative control: a planted wrong digest is refused with both digests, and + // the key is then unusable until a load verifies. if err := w.Load("x", path, 1, rows, strings.Repeat("0", 64)); err == nil || !strings.Contains(err.Error(), sha) { t.Errorf("wrong digest: %v", err) } + if _, err := w.Timer("x", fn)(1); err == nil { + t.Error("timed a key whose digest did not verify") + } + if err := w.Load("x", path, 1, rows, sha); err != nil { + t.Fatal(err) + } var nf int got, err := w.Evaler("x", fn, filepath.Join(dir, "o.f64"), func(n, _ int) { nf = n })() if err != nil { diff --git a/review_test.go b/review_test.go new file mode 100644 index 0000000..7275d35 --- /dev/null +++ b/review_test.go @@ -0,0 +1,196 @@ +package refbench + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestAccuracyWithoutRefsRoundTrips(t *testing.T) { + // A case with Accuracy and no Refs needs schema 2 (go_accuracy is a schema 2 + // field): the same predicate decides what Finish writes and what ReadReport + // accepts. + var log []string + c := multiCase(&log) + c.Refs = nil + c.Go = spinFor(12000) + c.Accuracy.Budget = 3 + rep, err := Run(context.Background(), []Case{c}, Options{Rounds: 3, MinTime: time.Millisecond, Load: func() (float64, error) { return 1, nil }}) + if err != nil { + t.Fatal(err) + } + rec, err := RecordAllocs([]Case{c}, Options{}) + if err != nil { + t.Fatal(err) + } + rec.Rows[0].GoAccuracy = &AccResult{Coverage: 1, Points: 1} + for name, r := range map[string]*Report{"measured": rep, "with accuracy only": rec} { + if r.Rows[0].GoAccuracy == nil && r.Rows[0].GoBudget == 0 { + continue // the deferred record before accuracy is scored carries none + } + var b bytes.Buffer + if err := r.Write(&b); err != nil { + t.Fatal(err) + } + if r.Schema != SchemaMultiRef { + t.Errorf("%s: schema %d, want 2", name, r.Schema) + } + back, err := ReadReport(&b) + if err != nil { + t.Fatalf("%s: a report this build wrote cannot be read back: %v", name, err) + } + if back.Schema != SchemaMultiRef || back.Rows[0].GoAccuracy == nil { + t.Errorf("%s: read back as schema %d, accuracy %v", name, back.Schema, back.Rows[0].GoAccuracy) + } + if tab := string(Table(back)); !strings.Contains(tab, "Against other references") { + t.Errorf("%s: the accuracy is not in the table:\n%s", name, tab) + } + } + // And a row with none of the fields stays schema 1. + plain := sampleReport() + if plain.Rows[0].usesMultiRef() || plain.Schema != SchemaVersion { + t.Error("a plain row uses schema 2") + } +} + +func TestDisagreeIsBounded(t *testing.T) { + for _, a := range []AccResult{{Points: 2, Disagree: 3}, {Points: 2, Disagree: -1}} { + if a.valid() { + t.Errorf("%+v valid", a) + } + } + if !(AccResult{Points: 2, Disagree: 2}).valid() { + t.Error("all points disagreeing is valid") + } + if s := accCell(&AccResult{MaxULP: 1, MaxULPFin: 1, Coverage: 1, Points: 4, Disagree: 2}); !strings.Contains(s, "2 disagree") { + t.Errorf("cell = %q", s) + } +} + +func TestRBW1RefusesAnUnverifiedKey(t *testing.T) { + dir := t.TempDir() + path, rows, sha, err := WriteF64(dir, "x", []float64{1, 2}) + if err != nil { + t.Fatal(err) + } + bad := strings.Repeat("0", 64) + reply, sent := "ok "+sha, 0 + w := fake(t, dir, []string{"f"}, func(line string) string { + sent++ + if strings.HasPrefix(line, "L ") { + return reply + } + return "ok" + }) + mustRefuse := func(when string) { + t.Helper() + before := sent + if err := w.Warm("x", "f"); err == nil || !strings.Contains(err.Error(), "no verified load") { + t.Errorf("%s: Warm: %v", when, err) + } + if _, err := w.Time("x", "f", 1); err == nil { + t.Errorf("%s: Time accepted", when) + } + if _, err := w.Timer("x", "f")(1); err == nil { + t.Errorf("%s: Timer accepted", when) + } + if _, _, err := w.Eval("x", "f", filepath.Join(dir, "o")); err == nil { + t.Errorf("%s: Eval accepted", when) + } + if _, err := w.Evaler("x", "f", filepath.Join(dir, "o"), nil)(); err == nil { + t.Errorf("%s: Evaler accepted", when) + } + if sent != before { + t.Errorf("%s: %d requests reached the worker", when, sent-before) + } + } + // 1. A digest mismatch: nothing may be timed on that key. + reply = "ok " + bad + if err := w.Load("x", path, 1, rows, sha); err == nil { + t.Fatal("mismatch accepted") + } + mustRefuse("after a mismatch") + // 2. A good load verifies the key ... + reply = "ok " + sha + if err := w.Load("x", path, 1, rows, sha); err != nil { + t.Fatal(err) + } + if err := w.Warm("x", "f"); err != nil { + t.Fatalf("verified key refused: %v", err) + } + // 3. ... and a reload that fails afterwards un-verifies it, whatever the failure. + reply = "ok " + bad + if err := w.Load("x", path, 1, rows, sha); err == nil { + t.Fatal("mismatch accepted") + } + mustRefuse("after a failed reload") + reply = "ok " + sha + if err := w.Load("x", path, 1, rows, sha); err != nil { + t.Fatal(err) + } + if err := w.Load("x", "relative", 1, rows, sha); err == nil { + t.Fatal("bad path accepted") + } + mustRefuse("after a reload refused before it was sent") +} + +func TestRBW1Boundaries(t *testing.T) { + dir := t.TempDir() + var last string + w := fake(t, dir, []string{"f"}, func(line string) string { + last = line + if strings.HasPrefix(line, "L ") { + return "ok " + strings.Repeat("0", 64) + } + return "ok" + }) + zero := strings.Repeat("0", 64) + // The request line: exactly 4096 bytes goes, 4097 does not. + fixed := len("L x 1 1") + n := RBW1MaxLine - fixed - len(dir) - 1 + exact := filepath.Join(dir, strings.Repeat("a", n)) + if len(exact)+fixed != RBW1MaxLine { + t.Fatalf("setup: %d", len(exact)+fixed) + } + if err := w.Load("x", exact, 1, 1, zero); err != nil || len(last) != RBW1MaxLine { + t.Errorf("a 4096-byte request: %v (sent %d bytes)", err, len(last)) + } + last = "" + if err := w.Load("x", exact+"a", 1, 1, zero); err == nil || last != "" { + t.Errorf("a 4097-byte request: %v (sent %q)", err, last) + } + // The data bound: 256 MiB goes, 8 bytes more does not. + p := filepath.Join(dir, "a") + if err := w.Load("x", p, 1, RBW1MaxData/8, zero); err != nil { + t.Errorf("256 MiB refused: %v", err) + } + if err := w.Load("x", p, 1, RBW1MaxData/8+1, zero); err == nil { + t.Error("256 MiB + 8 bytes accepted") + } + if err := w.Load("x", p, 2, RBW1MaxData/16, zero); err != nil { + t.Errorf("2 columns of 128 MiB: %v", err) + } + // A loaded key: the output path of an evaluation must lie in the run directory. + if err := w.Load("x", p, 1, 1, zero); err != nil { + t.Fatal(err) + } + last = "" + if _, _, err := w.Eval("x", "f", filepath.Join(filepath.Dir(dir), "out")); err == nil || last != "" { + t.Errorf("eval outside the run directory: %v (sent %q)", err, last) + } +} + +func TestRoundWithMoreSamplesThanReferencesIsRefused(t *testing.T) { + rep := synth(10000) + rep.Rows[0].Rounds[0].RefsNs["x:extra"] = 1 // one sample too many + var b bytes.Buffer + if err := rep.Write(&b); err != nil { + t.Fatal(err) + } + if _, err := ReadReport(&b); err == nil || !strings.Contains(err.Error(), "one sample per reference") { + t.Errorf("err = %v", err) + } +} diff --git a/runner.go b/runner.go index 096f390..c670e04 100644 --- a/runner.go +++ b/runner.go @@ -234,9 +234,12 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { return row, err } // The sides, in the order of the registry: Go, the primary, the extra ones. - // Round r runs them rotated by r mod len(sides), so that every side takes - // every position once per len(sides) rounds. With two sides this is the - // alternation of schema 1 (Go first on even rounds). + // Round r runs them in the order williamsOrders(k)[r mod period]: every + // side takes every position equally often AND every ordered pair of sides + // is adjacent equally often, so that no side always follows the same other + // one (a plain rotation balances positions, not that first-order carryover). + // With two sides this is the alternation of schema 1 (Go first on even + // rounds). sides := []roundSide{ {id: SideGo, t: g, iters: row.GoIters}, {id: SidePrimary, t: f, iters: row.RefIters}, @@ -250,23 +253,23 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { sides = append(sides, roundSide{id: r.ID, t: r.Timer, iters: it}) } k := len(sides) + orders := williamsOrders(k) rounds := opt.Rounds - if k > 2 && rounds%k != 0 { - rounds += k - rounds%k + if p := len(orders); k > 2 && rounds%p != 0 { + rounds += p - rounds%p } for r := 0; r < rounds; { load, err := opt.gate(ctx) if err != nil { return row, err } - // Go is side 0 and the primary side 1: at position (i - r mod k) mod k. - goPos, primaryPos := (k-r%k)%k, (k+1-r%k)%k - rd := Round{LoadBefore: load, GoFirst: goPos < primaryPos} + order := orders[r%len(orders)] + rd := Round{LoadBefore: load, GoFirst: before(order, 0, 1)} if k > 2 { rd.RefsNs = make(map[string]float64, k-2) } for p := 0; p < k; p++ { - s := sides[(r+p)%k] + s := sides[order[p]] d, err := s.t(s.iters) if err != nil { return row, err @@ -302,6 +305,56 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { return row, nil } +// williamsOrders returns the run orders of k sides over one period: a Williams +// design (a balanced Latin square). Row i is (s[j] + i) mod k for the zigzag +// s = 0, 1, k-1, 2, k-2, ...; for even k the k rows balance positions and +// every ordered pair of sides is adjacent exactly once; for odd k the rows and +// their reversals (2k rows) do. k = 2 gives {0,1}, {1,0}. +func williamsOrders(k int) [][]int { + s := make([]int, k) + lo, hi := 1, k-1 + for j := 1; j < k; j++ { + if j%2 == 1 { + s[j] = lo + lo++ + } else { + s[j] = hi + hi-- + } + } + var out [][]int + for i := 0; i < k; i++ { + row := make([]int, k) + for j := range row { + row[j] = (s[j] + i) % k + } + out = append(out, row) + } + if k%2 == 1 { + for i := 0; i < k; i++ { + rev := make([]int, k) + for j := range rev { + rev[j] = out[i][k-1-j] + } + out = append(out, rev) + } + } + return out +} + +// before reports whether side a runs before side b in order. +func before(order []int, a, b int) bool { + for _, s := range order { + if s == a { + return true + } + if s == b { + return false + } + } + return false +} + type roundSide struct { id string t Timer diff --git a/table.go b/table.go index 79b63dd..010728d 100644 --- a/table.go +++ b/table.go @@ -86,12 +86,15 @@ func refsSection(b *bytes.Buffer, r *Report) { } b.WriteString("\n") for _, row := range r.Rows { - if len(row.Refs) == 0 { + if !row.usesMultiRef() { continue } goNs, goUlp, best, ratio, label := "—", ulpCell(row.GoAccuracy), "—", "—", "deferred" if len(row.Rounds) > 0 { - goNs, best, ratio, label = fmt.Sprintf("%.4g", row.Go.Median), esc(sideName(row, row.Best)), fmt.Sprintf("%.2f", row.BestRatio), row.BestVerdict + goNs, label = fmt.Sprintf("%.4g", row.Go.Median), row.Verdict + if row.Best != "" { // a row with accuracy but no extra reference has no headline + best, ratio, label = esc(sideName(row, row.Best)), fmt.Sprintf("%.2f", row.BestRatio), row.BestVerdict + } if row.BestLabel != "" { label += ", " + row.BestLabel } @@ -153,7 +156,11 @@ func accCell(a *AccResult) string { if a == nil { return "no accuracy" } - return fmt.Sprintf("%.3g ulp, cov %.0f%%", float64(a.MaxULP), 100*a.Coverage) + s := fmt.Sprintf("%.3g ulp, fin %.3g, cov %.0f%%", float64(a.MaxULP), float64(a.MaxULPFin), 100*a.Coverage) + if a.Disagree > 0 { + s += fmt.Sprintf(", %d disagree", a.Disagree) + } + return s } func cell(s Summary) string { diff --git a/williams_test.go b/williams_test.go new file mode 100644 index 0000000..28aed9a --- /dev/null +++ b/williams_test.go @@ -0,0 +1,57 @@ +package refbench + +import "testing" + +func TestWilliamsOrdersAreBalanced(t *testing.T) { + for k := 2; k <= 9; k++ { + orders := williamsOrders(k) + want := k + if k%2 == 1 { + want = 2 * k + } + if len(orders) != want { + t.Fatalf("k=%d: %d orders, want %d", k, len(orders), want) + } + pos := map[[2]int]int{} // (side, position) -> count + pair := map[[2]int]int{} // (a, b adjacent, a before b) -> count + for _, o := range orders { + seen := map[int]bool{} + for p, s := range o { + if s < 0 || s >= k || seen[s] { + t.Fatalf("k=%d: %v is not a permutation", k, o) + } + seen[s] = true + pos[[2]int{s, p}]++ + if p > 0 { + pair[[2]int{o[p-1], s}]++ + } + } + } + for s := 0; s < k; s++ { + for p := 0; p < k; p++ { + if pos[[2]int{s, p}] != len(orders)/k { + t.Errorf("k=%d: side %d at position %d %d times, want %d", k, s, p, pos[[2]int{s, p}], len(orders)/k) + } + } + } + // Every ordered pair follows each other equally often: nobody always + // follows the same side (first-order carryover). + wantPair := len(orders) * (k - 1) / (k * (k - 1)) + for a := 0; a < k; a++ { + for b := 0; b < k; b++ { + if a != b && pair[[2]int{a, b}] != wantPair { + t.Errorf("k=%d: %d directly before %d %d times, want %d", k, a, b, pair[[2]int{a, b}], wantPair) + } + } + } + } + if o := williamsOrders(2); o[0][0] != 0 || o[1][0] != 1 { + t.Errorf("k=2 is not Go-first then reference-first: %v", o) + } + if before([]int{2, 3}, 0, 1) { + t.Error("before with neither side present") + } + if !before([]int{1, 0}, 1, 0) || before([]int{1, 0}, 0, 1) { + t.Error("before") + } +} diff --git a/workers/compiled/rbw1.cpp b/workers/compiled/rbw1.cpp index faf412c..2f805e8 100644 --- a/workers/compiled/rbw1.cpp +++ b/workers/compiled/rbw1.cpp @@ -26,6 +26,9 @@ #include #include #include +#ifdef _OPENMP +#include +#endif extern "C" { typedef void (*rbw1_fn)(int64_t n, const double *const *cols, double *out); @@ -152,6 +155,30 @@ rbw1_fn find(const std::string &name) { return nullptr; } +// threadInfo reports what the harness can observe, and says where it comes from: +// the OpenMP maximum when the worker is built with it, else (Linux) the number +// of threads of this process now, else an assumption ("assumed": the harness +// starts no thread, but a linked library that does so is NOT detected there). +std::string threadInfo() { +#ifdef _OPENMP + return "threads=" + std::to_string(omp_get_max_threads()) + " threads_source=omp"; +#elif defined(__linux__) + FILE *f = fopen("/proc/self/status", "r"); + if (f) { + char l[256]; + while (fgets(l, sizeof l, f)) { + long n; + if (sscanf(l, "Threads: %ld", &n) == 1) { + fclose(f); + return "threads=" + std::to_string(n) + " threads_source=proc"; + } + } + fclose(f); + } +#endif + return "threads=1 threads_source=assumed"; +} + void call(rbw1_fn f, Data &d) { f(d.rows, d.cols.data(), d.out.data()); g_sink = g_sink + d.out[0]; @@ -224,7 +251,7 @@ void handle(const std::string &line) { #endif for (char &ch : ver) if (ch == ' ') ch = '_'; - std::string s = "v compiler=" + ver + " threads=1 ldbl_mant_dig=" + std::to_string(LDBL_MANT_DIG); + std::string s = "v compiler=" + ver + " " + threadInfo() + " ldbl_mant_dig=" + std::to_string(LDBL_MANT_DIG); #ifdef RBW1_FLAGS std::string fl = RBW1_FLAGS; for (char &ch : fl) diff --git a/workers/julia/worker.jl b/workers/julia/worker.jl index eb49cd7..9e6569d 100644 --- a/workers/julia/worker.jl +++ b/workers/julia/worker.jl @@ -41,7 +41,8 @@ end function load(key, path, cols, rows) cols = parse(Int, cols); rows = parse(Int, rows) - (cols > 0 && rows > 0 && cols * rows * 8 <= MAXDATA) || error("size out of range") + # Each factor is bounded BEFORE the product is formed: Int64 arithmetic wraps. + (0 < cols <= 1 << 28 && 0 < rows <= 1 << 28 && cols * rows * 8 <= MAXDATA) || error("size out of range") inside(path) bytes = read(path) length(bytes) == cols * rows * 8 || error("file holds $(length(bytes)) bytes, want $(cols * rows * 8)") @@ -69,14 +70,19 @@ end function evalpoints!(f, out::Vector{Float64}, args::Tuple) raised = 0 + first = nothing for i in eachindex(out) out[i] = try Float64(f(map(a -> a[i], args)...)) - catch + catch e + first === nothing && (first = e) raised += 1 NaN end end + # A kernel that raises on EVERY point is a broken kernel (wrong arity, a + # typo), not a domain: refuse instead of reporting an all-NaN column. + raised == length(out) && error("the kernel raised on every point: " * oneline(sprint(showerror, first))) raised end From 3e3a308b8c90dd2d9c2220378986923b900c6880 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 17:36:17 +0100 Subject: [PATCH 09/12] Review round 2: no false "ahead", boundary tests found by mutsweep - Label: a Go with Disagree > 0 or lower coverage than the reference is never "ahead" (faster-less-accurate); the probe is a test. - Boundary tests: request line, 256 MiB, Load dimensions (no product, no division), WriteF64/ReadF64, Eval reply counts and out-path, ns at 2^63, Budget at 0, version pairs at 256, printable ASCII edges. - Close reports the process's exit status; the fake worker's reply pipe is buffered so a client defect fails an assertion instead of hanging. - mutsweep boundary: 29 mutations, 29 caught, 0 survived, 0 hung. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- multiref.go | 7 ++ multiref_test.go | 8 +- rbw1.go | 11 +- rbw1_test.go | 11 +- round2_test.go | 285 +++++++++++++++++++++++++++++++++++++++++++++++ runner.go | 16 +-- 6 files changed, 321 insertions(+), 17 deletions(-) create mode 100644 round2_test.go diff --git a/multiref.go b/multiref.go index a77c673..45d5db4 100644 --- a/multiref.go +++ b/multiref.go @@ -169,6 +169,13 @@ func Label(verdict string, goAcc, refAcc *AccResult, budget float64) string { if verdict == "" || goAcc == nil || refAcc == nil { return "" } + // "ahead" promises at least as accurate. MaxULPFin is the measure against a + // reference, but a Go that returns NaN where a number is expected + // (Disagree), or evaluates fewer points than the reference (Coverage), is + // not ahead on accuracy however small its finite error is. + if goAcc.Disagree > 0 || goAcc.Coverage < refAcc.Coverage { + return LabelFasterLessAccurate + } if goAcc.MaxULPFin <= refAcc.MaxULPFin { return LabelAhead } diff --git a/multiref_test.go b/multiref_test.go index 8268780..46a09cd 100644 --- a/multiref_test.go +++ b/multiref_test.go @@ -187,8 +187,12 @@ func TestLabel(t *testing.T) { {Slower, inf1, inf1, 0, LabelBehind, "both infinite, same finite error: behind (was always behind)"}, {Slower, inf1, inf3, 0, LabelTradeoff, "Go's own max is infinite: the finite error still decides"}, {Slower, inf3, inf1, 0, LabelBehind, "ref more accurate on finite points"}, - {Faster, inf1, inf3, 0, LabelAhead, "finite error decides ahead"}, - {Faster, inf3, inf1, 0, LabelFasterLessAccurate, "finite error decides faster-less-accurate"}, + {Faster, one, inf3, 0, LabelAhead, "the finite error decides against a reference that disagrees"}, + {Faster, one, inf1, 0, LabelAhead, "tie on the finite error, the reference's own disagreement does not block"}, + {Faster, two, inf1, 0, LabelFasterLessAccurate, "the finite error decides"}, + {Faster, inf1, inf3, 0, LabelFasterLessAccurate, "Go's own disagreement blocks ahead"}, + {Faster, &AccResult{MaxULPFin: 1, Coverage: 0.9}, &AccResult{MaxULPFin: 1, Coverage: 1}, 0, LabelFasterLessAccurate, "Go evaluates fewer points"}, + {Faster, &AccResult{MaxULPFin: 1, Coverage: 1}, &AccResult{MaxULPFin: 1, Coverage: 0.9}, 0, LabelAhead, "Go evaluates more points"}, } for _, c := range cases { if got := Label(c.v, c.g, c.r, c.budget); got != c.want { diff --git a/rbw1.go b/rbw1.go index 7a9f047..2e806c6 100644 --- a/rbw1.go +++ b/rbw1.go @@ -39,8 +39,6 @@ const ( RBW1MaxLine = 4096 // RBW1MaxData bounds the bytes of one loaded argument file (256 MiB). RBW1MaxData = 256 << 20 - // rbw1MaxDim bounds cols and rows of a request. - rbw1MaxDim = 1 << 28 // rbw1MaxVersions bounds the pairs of a version reply. rbw1MaxVersions = 256 ) @@ -256,9 +254,6 @@ func (w *RBW1) ask(parts ...string) (RBW1Reply, error) { } func (w *RBW1) check(key, fn string) error { - if !rbw1KeyPattern.MatchString(key) { - return fmt.Errorf("rbw1: key %q is not a plain identifier", key) - } if !w.funcs[fn] { return fmt.Errorf("rbw1: function %q is not in the worker's table", fn) } @@ -293,7 +288,11 @@ func (w *RBW1) Load(key, path string, cols, rows int, want string) error { if err := w.inDir(path); err != nil { return err } - if cols <= 0 || rows <= 0 || cols > rbw1MaxDim || rows > rbw1MaxDim || int64(cols)*int64(rows)*8 > RBW1MaxData { + // cols*rows*8 <= RBW1MaxData: each factor is bounded before the product is + // formed in int64, so nothing overflows, on 32 bits or 64 (and nothing + // divides: a zero factor is refused by its own comparison). + const maxCells = RBW1MaxData / 8 + if cols <= 0 || rows <= 0 || int64(cols) > maxCells || int64(rows) > maxCells || int64(cols)*int64(rows) > maxCells { return fmt.Errorf("rbw1: %d x %d float64 is outside 1..%d bytes", cols, rows, RBW1MaxData) } r, err := w.ask("L", key, path, strconv.Itoa(cols), strconv.Itoa(rows)) diff --git a/rbw1_test.go b/rbw1_test.go index 39405c8..39aa08f 100644 --- a/rbw1_test.go +++ b/rbw1_test.go @@ -88,7 +88,13 @@ func FuzzParseRBW1Reply(f *testing.F) { func fake(t *testing.T, dir string, funcs []string, respond func(line string) string) *RBW1 { t.Helper() inR, inW := io.Pipe() - outR, outW := io.Pipe() + // The replies go through an OS pipe, which buffers: a client that never + // reads a reply (a defect under test) then fails an assertion instead of + // deadlocking the worker goroutine against it. + outR, outW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } go func() { sc := bufio.NewScanner(inR) for sc.Scan() { @@ -414,6 +420,9 @@ func TestHelperRBW1(t *testing.T) { fmt.Println("ok") } } + if mode == "exit3" { + os.Exit(3) // stdin closed: leave with a status Close must report + } os.Exit(0) } diff --git a/round2_test.go b/round2_test.go new file mode 100644 index 0000000..4a1c7de --- /dev/null +++ b/round2_test.go @@ -0,0 +1,285 @@ +package refbench + +import ( + "context" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestFasterButNaNOnTenPercentIsNotAhead(t *testing.T) { + // The reviewer's probe: Go has a tiny finite error but is NaN on 10% of the + // points where the reference returns numbers; "ahead" would promise "at + // least as accurate". + goAcc := &AccResult{MaxULP: ULP(math.Inf(1)), MaxULPFin: 0.5, Coverage: 0.9, Disagree: 1, Points: 10} + ref := &AccResult{MaxULP: 1, MaxULPFin: 1, Coverage: 1, Points: 10} + if got := Label(Faster, goAcc, ref, 0); got != LabelFasterLessAccurate { + t.Errorf("Label = %q, want %q", got, LabelFasterLessAccurate) + } + if got := Label(Tie, goAcc, ref, 0); got != LabelFasterLessAccurate { + t.Errorf("tie: Label = %q", got) + } +} + +func TestPrintableASCIIBoundaries(t *testing.T) { + for c, want := range map[byte]bool{0x00: false, 0x1f: false, 0x20: true, 0x41: true, 0x7e: true, 0x7f: false, 0x80: false, 0xff: false} { + if got := printableASCII([]byte{'a', c, 'b'}); got != want { + t.Errorf("byte %#x: printable = %v, want %v", c, got, want) + } + } + if !printableASCII(nil) { + t.Error("empty is printable") + } + // Through the parser: space (0x20) is the separator, '~' (0x7e) is allowed. + if r, err := ParseRBW1Reply([]byte("ok ~ a~")); err != nil || len(r.Args) != 2 { + t.Errorf("0x7e: %v %v", r, err) + } + if _, err := ParseRBW1Reply([]byte("ok \x7f")); err == nil { + t.Error("0x7f accepted") + } + if _, err := ParseRBW1Reply([]byte("ok \x1f")); err == nil { + t.Error("0x1f accepted") + } +} + +func TestVersionPairsBoundary(t *testing.T) { + pairs := func(n int) []byte { + var b strings.Builder + b.WriteString("v") + for i := 0; i < n; i++ { + b.WriteString(" k" + strconv.Itoa(i) + "=1") + } + return []byte(b.String()) + } + if r, err := ParseRBW1Reply(pairs(rbw1MaxVersions)); err != nil || len(r.Versions) != rbw1MaxVersions { + t.Errorf("256 pairs: %v", err) + } + if _, err := ParseRBW1Reply(pairs(rbw1MaxVersions + 1)); err == nil { + t.Error("257 pairs accepted") + } + if r, err := ParseRBW1Reply(pairs(1)); err != nil || len(r.Versions) != 1 { + t.Errorf("1 pair: %v", err) + } + if _, err := ParseRBW1Reply([]byte("v")); err == nil { + t.Error("no pairs accepted") + } +} + +func TestTimeBoundary(t *testing.T) { + dir := t.TempDir() + reply := "" + w := fake(t, dir, []string{"f"}, func(string) string { return reply }) + // 2^63 ns does not fit a time.Duration; the largest float below it does. + reply = "ns 9223372036854775808" + if _, err := w.Time("x", "f", 1); err == nil || !strings.Contains(err.Error(), "overflows") { + t.Errorf("2^63: %v", err) + } + reply = "ns 9223372036854774784" + if d, err := w.Time("x", "f", 1); err != nil || d <= 0 { + t.Errorf("2^63 - 1024: %v %v", d, err) + } + reply = "ns 0" + if d, err := w.Time("x", "f", 1); err != nil || d != 0 { + t.Errorf("0: %v %v", d, err) + } +} + +func TestEvalReplyBoundaries(t *testing.T) { + dir := t.TempDir() + reply := "" + w := fake(t, dir, []string{"f"}, func(string) string { return reply }) + out := filepath.Join(dir, "o") + for _, c := range []struct { + reply string + a, b int + ok bool + }{ + {"ok 0 0", 0, 0, true}, {"ok 1 0", 1, 0, true}, {"ok 0 1", 0, 1, true}, {"ok 7 9", 7, 9, true}, + {"ok -1 0", 0, 0, false}, {"ok 0 -1", 0, 0, false}, {"ok 1", 0, 0, false}, {"ok 1 2 3", 0, 0, false}, + } { + reply = c.reply + a, b, err := w.Eval("x", "f", out) + if (err == nil) != c.ok || (c.ok && (a != c.a || b != c.b)) { + t.Errorf("%q: %d %d %v", c.reply, a, b, err) + } + } + // The worker's own error is what comes back, not a generic one. + reply = "err domain error at point 3" + if _, _, err := w.Eval("x", "f", out); err == nil || !strings.Contains(err.Error(), "domain error at point 3") { + t.Errorf("worker error lost: %v", err) + } + // The out path is refused on the Go side before anything is sent. + sent := 0 + w2 := fake(t, dir, []string{"f"}, func(string) string { sent++; return "ok 0 0" }) + for _, p := range []string{"relative", filepath.Join(filepath.Dir(dir), "o"), dir, dir + string(filepath.Separator) + ".." + string(filepath.Separator) + "o"} { + if _, _, err := w2.Eval("x", "f", p); err == nil || !strings.Contains(err.Error(), "inside") { + t.Errorf("out path %q: %v", p, err) + } + } + if sent != 0 { + t.Errorf("%d requests reached the worker", sent) + } +} + +func TestLoadDimensionBoundaries(t *testing.T) { + dir := t.TempDir() + var last string + w := fake(t, dir, []string{"f"}, func(l string) string { last = l; return "ok " + strings.Repeat("0", 64) }) + zero, p := strings.Repeat("0", 64), filepath.Join(dir, "a") + const maxRows = RBW1MaxData / 8 + for _, c := range []struct { + cols, rows int + ok bool + }{ + {1, 1, true}, {0, 1, false}, {1, 0, false}, {0, 0, false}, {-1, 1, false}, {1, -1, false}, + {1, maxRows, true}, {1, maxRows + 1, false}, + {2, maxRows / 2, true}, {2, maxRows/2 + 1, false}, + {3, maxRows / 3, true}, {3, maxRows/3 + 1, false}, + {maxRows, 1, true}, {maxRows + 1, 1, false}, + {math.MaxInt32, math.MaxInt32, false}, {math.MaxInt32, 1, false}, + } { + last = "" + err := w.Load("x", p, c.cols, c.rows, zero) + if (err == nil) != c.ok || (c.ok && last == "") || (!c.ok && last != "") { + t.Errorf("%d x %d: err %v, sent %q", c.cols, c.rows, err, last) + } + } +} + +func TestWriteF64NameAndLimit(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "run") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"../escape", "a/b", "", "a b", "a.b", strings.Repeat("a", 65)} { + _, _, _, err := WriteF64(dir, name, []float64{1}) + if err == nil || !strings.Contains(err.Error(), "plain identifier") { + t.Errorf("name %q: %v", name, err) + } + } + for _, f := range []string{filepath.Join(parent, "escape.f64"), filepath.Join(dir, "a", "b.f64")} { + if _, err := os.Stat(f); err == nil { + t.Errorf("%s was written", f) + } + } + if _, _, _, err := WriteF64(dir, strings.Repeat("a", 64), []float64{1}); err != nil { + t.Errorf("a 64-character name: %v", err) + } + old := dataLimit + defer func() { dataLimit = old }() + dataLimit = 16 // two float64 + if _, _, _, err := WriteF64(dir, "two", []float64{1, 2}); err != nil { + t.Errorf("exactly at the limit: %v", err) + } + if _, _, _, err := WriteF64(dir, "three", []float64{1, 2, 3}); err == nil { + t.Error("one value over the limit accepted") + } + if _, _, _, err := WriteF64(dir, "wide", []float64{1}, []float64{2}); err != nil { + t.Errorf("two columns of one row at the limit: %v", err) + } + if _, _, _, err := WriteF64(dir, "wide3", []float64{1}, []float64{2}, []float64{3}); err == nil { + t.Error("three columns over the limit accepted") + } + // ReadF64 bounds. + empty := filepath.Join(dir, "empty.f64") + if err := os.WriteFile(empty, nil, 0o600); err != nil { + t.Fatal(err) + } + if got, err := ReadF64(empty, 0); err != nil || len(got) != 0 { + t.Errorf("n = 0: %v %v", got, err) + } + two := filepath.Join(dir, "two.f64") + if got, err := ReadF64(two, 2); err != nil || len(got) != 2 { + t.Errorf("n at the limit: %v %v", got, err) + } + if _, err := ReadF64(two, 3); err == nil || !strings.Contains(err.Error(), "out of range") { + t.Errorf("n over the limit: %v", err) + } + if _, err := ReadF64(two, -1); err == nil || !strings.Contains(err.Error(), "out of range") { + t.Errorf("n = -1: %v", err) + } +} + +func TestValidateBudgetBoundary(t *testing.T) { + score := func([]float64) AccResult { return AccResult{} } + eval := func() ([]float64, error) { return nil, nil } + mk := func(b float64) Case { + return Case{Name: "a", Func: "A", Parity: ParityNone, Go: func(int) {}, Accuracy: &Accuracy{Go: eval, Score: score, Budget: b}} + } + for b, ok := range map[float64]bool{0: true, 1e-300: true, 4: true, -1e-300: false, -1: false, math.NaN(): false} { + if err := Validate([]Case{mk(b)}); (err == nil) != ok { + t.Errorf("Budget %v: %v", b, err) + } + } +} + +func TestRunWithNothingToMeasureIsNotAnError(t *testing.T) { + var log []string + c := multiCase(&log) + rep, err := Run(context.Background(), []Case{c}, Options{Only: map[string]bool{"nothing": true}, Load: func() (float64, error) { return 1, nil }}) + if err != nil || len(rep.Rows) != 0 { + t.Errorf("an empty resume: %v %+v", err, rep) + } + // And a gate that never opens, with rows to measure, is an error. + _, err = Run(context.Background(), []Case{c}, Options{OnLoad: OnLoadRefuse, Load: func() (float64, error) { return 99, nil }}) + if err == nil { + t.Error("a closed gate with a row to measure gave a report") + } +} + +func TestCloseWaitsForTheProcessAndReportsItsExit(t *testing.T) { + helperRBW1(t, "exit3") + w, err := StartRBW1(context.Background(), RBW1Config{Dir: t.TempDir(), Funcs: []string{"f"}}, os.Environ(), "x") + if err != nil { + t.Fatal(err) + } + if err := w.Close(); err == nil || !strings.Contains(err.Error(), "exit status 3") { + t.Errorf("Close = %v, want the process's exit status", err) + } + // A worker with no process (a pair of streams) just closes its input. + if err := fake(t, t.TempDir(), []string{"f"}, func(string) string { return "ok" }).Close(); err != nil { + t.Errorf("Close of a stream worker: %v", err) + } +} + +func TestRefRowsAndLabels(t *testing.T) { + if rows := (Case{}).refRows(); rows != nil { + t.Errorf("a case with no Refs has refRows = %#v, want nil", rows) + } + if got := (RefRow{ID: "a:b", Name: "Nice"}).label(); got != "Nice" { + t.Errorf("label = %q", got) + } + if got := (RefRow{ID: "a:b"}).label(); got != "a:b" { + t.Errorf("label = %q", got) + } +} + +func TestNilAccuracyCellsAndScoring(t *testing.T) { + if ulpCell(nil) != "—" || covCell(nil) != "—" || accCell(nil) != "no accuracy" { + t.Errorf("nil cells: %q %q %q", ulpCell(nil), covCell(nil), accCell(nil)) + } + // No Accuracy: nothing is scored, nothing recorded. + var log []string + c := multiCase(&log) + c.Go = spinFor(12000) + c.Accuracy, c.RefEval = nil, nil + row := c.row(0) + if err := scoreAccuracy(c, &row); err != nil || row.GoAccuracy != nil || row.PrimaryAccuracy != nil || row.GoBudget != 0 || row.Refs[0].Accuracy != nil { + t.Errorf("without Accuracy: %v %+v", err, row) + } + // Accuracy, but a side without Eval: that side has none, the others do. + c = multiCase(&log) + c.RefEval = nil + c.Refs[0].Eval = nil + row = c.row(0) + if err := scoreAccuracy(c, &row); err != nil { + t.Fatal(err) + } + if row.GoAccuracy == nil || row.PrimaryAccuracy != nil || row.Refs[0].Accuracy != nil || row.GoBudget != 1 { + t.Errorf("a side without Eval: %+v", row) + } +} diff --git a/runner.go b/runner.go index c670e04..7581c81 100644 --- a/runner.go +++ b/runner.go @@ -268,8 +268,8 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { if k > 2 { rd.RefsNs = make(map[string]float64, k-2) } - for p := 0; p < k; p++ { - s := sides[order[p]] + for _, side := range order { + s := sides[side] d, err := s.t(s.iters) if err != nil { return row, err @@ -311,20 +311,20 @@ func runCase(ctx context.Context, c Case, opt Options) (Row, error) { // every ordered pair of sides is adjacent exactly once; for odd k the rows and // their reversals (2k rows) do. k = 2 gives {0,1}, {1,0}. func williamsOrders(k int) [][]int { - s := make([]int, k) + s := []int{0} lo, hi := 1, k-1 - for j := 1; j < k; j++ { - if j%2 == 1 { - s[j] = lo + for len(s) < k { + if len(s)%2 == 1 { + s = append(s, lo) lo++ } else { - s[j] = hi + s = append(s, hi) hi-- } } var out [][]int for i := 0; i < k; i++ { - row := make([]int, k) + row := make([]int, len(s)) // len(s) == k; a zigzag of any other length is not a permutation for j := range row { row[j] = (s[j] + i) % k } From 76dcb0437b001cdbfa81b9157e264963c1a1c0ac Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 17:38:27 +0100 Subject: [PATCH 10/12] Tests: a Windows clock that times the Go loop at 0 ns is not a ratio defect TestRunVerdictAndInvalid asserted ratio > 0 and failed on Windows once (a coarse clock, ratio 0). It checks the ratio structurally now. Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- core_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core_test.go b/core_test.go index ebae431..4dd6f26 100644 --- a/core_test.go +++ b/core_test.go @@ -182,7 +182,9 @@ func TestRunVerdictAndInvalid(t *testing.T) { if err != nil { t.Fatal(err) } - if rep.Rows[0].Ratio <= 0 { + // Structural only: Windows' coarse clock can time the Go loop at 0 ns + // (ratio 0); a NaN, an infinity or a negative ratio is still a defect. + if r := rep.Rows[0].Ratio; !(r >= 0) || math.IsInf(r, 0) { t.Errorf("ratio = %v", rep.Rows[0].Ratio) } } From 37577f5c4868dc2b3f218d18c2bd1961ee3527c8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 17:53:49 +0100 Subject: [PATCH 11/12] Tests: assert the refusal of every nil and index guard Refs #15 Co-Authored-By: Claude Sonnet 5.5 --- a_guards_test.go | 163 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 a_guards_test.go diff --git a/a_guards_test.go b/a_guards_test.go new file mode 100644 index 0000000..c6c3dc4 --- /dev/null +++ b/a_guards_test.go @@ -0,0 +1,163 @@ +package refbench + +import ( + "context" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// This file runs first (its name sorts before the others). Each test reaches a +// guard that protects a nil or an index, and asserts the REFUSAL. A mutant that +// removes the guard then fails an assertion here, instead of the suite going +// red only because the code after the missing guard crashes (a guard held by a +// crash is not held). Where the mutant necessarily crashes inside the call, the +// call is wrapped by mustNotPanic, which turns the crash into a failure that +// names the missing guard. + +func mustNotPanic(t *testing.T, what string, f func()) { + t.Helper() + defer func() { + if r := recover(); r != nil { + t.Errorf("%s panicked: guard missing (%v)", what, r) + } + }() + f() +} + +func TestGuardReplyShapes(t *testing.T) { + for _, in := range []string{"ns", "ns 1 2"} { + mustNotPanic(t, "ParseRBW1Reply("+in+")", func() { + if _, err := ParseRBW1Reply([]byte(in)); err == nil { + t.Errorf("%q accepted", in) + } + }) + } + dir := t.TempDir() + path, rows, sha, err := WriteF64(dir, "x", []float64{1}) + if err != nil { + t.Fatal(err) + } + reply := "ok" + w := fake(t, dir, []string{"f"}, func(string) string { return reply }) + mustNotPanic(t, "Load with an 'ok' reply that has no digest", func() { + if err := w.Load("x", path, 1, rows, sha); err == nil { + t.Error("a load reply without a digest accepted") + } + }) + w.rows["x"] = 1 + mustNotPanic(t, "Eval with an 'ok' reply that has no counts", func() { + if _, _, err := w.Eval("x", "f", filepath.Join(dir, "o")); err == nil { + t.Error("an eval reply without counts accepted") + } + }) +} + +func TestGuardFiles(t *testing.T) { + dir := t.TempDir() + mustNotPanic(t, "WriteF64 with no column", func() { + if _, _, _, err := WriteF64(dir, "a"); err == nil { + t.Error("no column accepted") + } + }) + mustNotPanic(t, "WriteF64 with an empty column", func() { + if _, _, _, err := WriteF64(dir, "a", []float64{}); err == nil { + t.Error("an empty column accepted") + } + }) + mustNotPanic(t, "ReadF64 of a missing file", func() { + if _, err := ReadF64(filepath.Join(dir, "missing.f64"), 1); err == nil { + t.Error("a missing file read") + } + }) +} + +func TestGuardStarts(t *testing.T) { + old := execCommand + defer func() { execCommand = old }() + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { + return exec.CommandContext(ctx, "/nonexistent/refbench-guard") + } + mustNotPanic(t, "StartRBW1 with a bad run directory", func() { + if _, err := StartRBW1(context.Background(), RBW1Config{Dir: "relative", Funcs: []string{"f"}}, nil, "x"); err == nil { + t.Error("a relative run directory accepted") + } + }) + mustNotPanic(t, "StartJulia when the worker cannot start", func() { + _, _, err := StartJulia(context.Background(), JuliaConfig{Pkgx: "pkgx", Version: "1.13.3", Project: "/p", Kernels: "/k.jl", Depot: "/d", Dir: t.TempDir(), Funcs: []string{"f"}}) + if err == nil { + t.Error("a worker that cannot start accepted") + } + }) +} + +func TestGuardLabelsAndRefIDs(t *testing.T) { + acc := &AccResult{MaxULPFin: 1, Coverage: 1} + mustNotPanic(t, "Label without Go accuracy", func() { + if got := Label(Faster, nil, acc, 0); got != "" { + t.Errorf("label = %q", got) + } + }) + mustNotPanic(t, "Label without reference accuracy", func() { + if got := Label(Faster, acc, nil, 0); got != "" { + t.Errorf("label = %q", got) + } + }) + mustNotPanic(t, "Label without a verdict", func() { + if got := Label("", acc, acc, 0); got != "" { + t.Errorf("label = %q", got) + } + }) + a := []RefRow{{ID: "a:x"}, {ID: "b:y"}} + mustNotPanic(t, "sameRefIDs of lists of different lengths", func() { + if sameRefIDs(a, a[:1]) || sameRefIDs(a[:1], a) { + t.Error("lists of different lengths are the same") + } + }) +} + +func TestGuardRunnerNils(t *testing.T) { + var log []string + c := multiCase(&log) + c.Ref = nil + mustNotPanic(t, "Run of a case without a primary timer", func() { + if _, err := Run(context.Background(), []Case{c}, Options{}); err == nil || !strings.Contains(err.Error(), "no reference timer") { + t.Errorf("err = %v", err) + } + }) + c = multiCase(&log) + c.Accuracy, c.RefEval = nil, nil + row := c.row(0) + mustNotPanic(t, "scoreAccuracy without Accuracy", func() { + if err := scoreAccuracy(c, &row); err != nil || row.GoAccuracy != nil { + t.Errorf("%v %+v", err, row) + } + }) + c = multiCase(&log) + c.RefEval, c.Refs[0].Eval = nil, nil + row = c.row(0) + mustNotPanic(t, "scoreAccuracy of sides without Eval", func() { + if err := scoreAccuracy(c, &row); err != nil || row.PrimaryAccuracy != nil || row.Refs[0].Accuracy != nil || row.GoAccuracy == nil { + t.Errorf("%v %+v", err, row) + } + }) +} + +func TestGuardNilCells(t *testing.T) { + mustNotPanic(t, "ulpCell(nil)", func() { + if ulpCell(nil) != "—" { + t.Error("ulpCell(nil)") + } + }) + mustNotPanic(t, "covCell(nil)", func() { + if covCell(nil) != "—" { + t.Error("covCell(nil)") + } + }) + mustNotPanic(t, "accCell(nil)", func() { + if accCell(nil) != "no accuracy" { + t.Error("accCell(nil)") + } + }) +} From cb707787d1ef7fe1c3c25481e42ecb445a872938 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 13 Nov 2026 18:00:51 +0100 Subject: [PATCH 12/12] Tests: StartRBW1's bad-config refusal with a process that can start --- a_guards_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/a_guards_test.go b/a_guards_test.go index c6c3dc4..1b3e479 100644 --- a/a_guards_test.go +++ b/a_guards_test.go @@ -74,16 +74,17 @@ func TestGuardFiles(t *testing.T) { } func TestGuardStarts(t *testing.T) { - old := execCommand - defer func() { execCommand = old }() - execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { - return exec.CommandContext(ctx, "/nonexistent/refbench-guard") - } + // A process that CAN start: without the refusal of the bad configuration it + // would be started and the worker would be nil. + helperRBW1(t, "ok") mustNotPanic(t, "StartRBW1 with a bad run directory", func() { if _, err := StartRBW1(context.Background(), RBW1Config{Dir: "relative", Funcs: []string{"f"}}, nil, "x"); err == nil { t.Error("a relative run directory accepted") } }) + execCommand = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { + return exec.CommandContext(ctx, "/nonexistent/refbench-guard") + } mustNotPanic(t, "StartJulia when the worker cannot start", func() { _, _, err := StartJulia(context.Background(), JuliaConfig{Pkgx: "pkgx", Version: "1.13.3", Project: "/p", Kernels: "/k.jl", Depot: "/d", Dir: t.TempDir(), Funcs: []string{"f"}}) if err == nil {