From fa6f6afd86fd97e63d1400615b098da5d0365897 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 9 Nov 2026 21:25:12 +0100 Subject: [PATCH] =?UTF-8?q?conformance:=20cmd/specroll=20=E2=80=94=20roll?= =?UTF-8?q?=20a=20specdelta=20report=20up=20to=20the=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit specdelta answers per FILE. "What is left to reach parity with MRI?" is a question about MODULES, and the two are far apart: one run of library/ produced 497 rows with 189 moved, and the shape of the debt -- 41 bigdecimal files carrying 180 of the missing examples, 5 coverage files carrying 53 more and not loading at all -- is not visible by reading 497 lines. Summarising them by eye is a count done by eye. It carries two controls, and both earned their place during this change. ⛔ IT CHECKS ITS ARITHMETIC AGAINST THE REPORT'S OWN FOOTER. specdelta prints "497 file(s): control 1370 passing, treatment 736 passing, -634"; specroll recomputes that from the rows it parsed and REFUSES when they disagree. A rollup is exactly where a dropped row is invisible: the table still looks tidy, the modules are still sorted, and the only symptom is a total nobody re-derives. ⛔ THE INVARIANT DIFFERS BETWEEN THE TWO REPORT SHAPES. A full report must match on all three numbers. A -quiet report prints only the files that MOVED, so its rows cannot match a footer counting every file walked -- what must match is the DELTA, since an omitted file contributes 0 to it. Every report this tool was built to read is a quiet one, so judging them by the full-report rule would have called each of them corrupt. The test for that case is TestAQuietReportIsReconciledByItsDelta. The first run of the test suite found a real sign error from the same family: specdelta's delta is TREATMENT MINUS CONTROL, the gap column here is the other way round on purpose, and reconciling with the wrong one reported every correct quiet report as off by twice the gap. ⛔ IT SEPARATES "BEHIND" FROM "ABSENT". A file the treatment cannot load is counted in its own column with the examples it costs, because "implement Coverage" and "fix 29 Date examples" are different work and summing them hides which is which. ⛔ IT STATES WHAT ITS COLUMNS COVER. With a quiet report in the mix the MRI and rbgo columns count only the moved files; the gap column is exact either way, which is why the table sorts by the gap and the sentence is printed rather than left as a footnote. A full rollup does not carry it (TestAFullRollupDoesNotCarryTheQuietCaveat): a caveat printed always is a caveat nobody reads. Style: stdlib flag, like its four neighbours in this directory (specdelta, ratchet, phantom, oracle). These are repository scripts, not released CLIs, and cobra for one of five would be the odd one out in the directory where they are read side by side. Co-Authored-By: Claude Opus 5 --- .../conformance/rubyspec/cmd/specroll/main.go | 338 ++++++++++++++++++ .../rubyspec/cmd/specroll/main_test.go | 175 +++++++++ 2 files changed, 513 insertions(+) create mode 100644 scripts/conformance/rubyspec/cmd/specroll/main.go create mode 100644 scripts/conformance/rubyspec/cmd/specroll/main_test.go diff --git a/scripts/conformance/rubyspec/cmd/specroll/main.go b/scripts/conformance/rubyspec/cmd/specroll/main.go new file mode 100644 index 00000000..7ccfd340 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/specroll/main.go @@ -0,0 +1,338 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +// Command specroll rolls one or more specdelta reports up to the MODULE. +// +// Why it exists. "What is left to reach parity with MRI?" is a question about +// modules -- Date, coverage, socket -- and specdelta answers it per FILE. One +// run of `library/` produced 497 rows, 189 of them moved; the shape of the debt +// (two Date files carry 53 of the 634 missing examples, five coverage files +// carry 53 more and do not load at all) is not visible by reading 497 lines, and +// a human summarising them by eye is a count done by eye. +// +// ⛔ IT CHECKS ITS OWN ARITHMETIC AGAINST THE REPORT'S. specdelta prints its own +// footer: +// +// 497 file(s): control 1370 passing, treatment 736 passing, -634 +// +// specroll recomputes those three numbers from the rows it parsed and REFUSES +// when they disagree. A rollup is exactly where a dropped row is invisible: the +// table still looks tidy, the modules are still sorted, and the only symptom is +// a total nobody re-derives. The judge and the subject must count alike. +// +// ⛔ IT SEPARATES "BEHIND" FROM "ABSENT". A file the treatment cannot load at +// all is reported by specdelta as DID NOT RUN, and its whole count is missing +// for a reason that is not a failing example -- a missing require, a missing +// class. Those are counted in their own column, because "implement Coverage" +// and "fix 29 Date examples" are different work and summing them hides which. +// +// ⛔ IT IS NOT A SECOND MEASUREMENT. It reads what specdelta measured and says +// nothing specdelta did not; a module missing from the reports is missing here. +// The population is the reports named on the command line, and the footer says +// which files it read, so a rollup of half a corpus cannot be read as a whole. +// +// Style note: stdlib `flag`, like its four neighbours in this directory +// (specdelta, ratchet, phantom, oracle). These are repository scripts, not +// released CLIs, and a cobra dependency for one of five would be the odd one +// out in the directory that matters most for reading them side by side. +// +// Usage: +// +// specroll [-depth N] [-top N] ... +// +// Exit status: 0 the rollup printed and reconciled, 1 a report's own totals +// disagree with the rows it contains, 2 nothing could be read. +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "path" + "regexp" + "sort" + "strconv" + "strings" +) + +// A specdelta per-file row: +// +// library/date/parse_spec.rb 54 22 -32 ⛔ fail+error 0 → 32 +// +// Anchored at column zero so the indented "⛔ WENT RED:" lines beneath a row -- +// which also contain digits and arrows -- can never be read as rows. +var rowRe = regexp.MustCompile(`^(\S+\.rb)\s+(\d+)\s+(\d+)\s+([+-]\d+)\s*(.*)$`) + +// specdelta's footer, the control this tool judges itself against. +var footerRe = regexp.MustCompile(`^(\d+) file\(s\): control (\d+) passing, treatment (\d+) passing, ([+-]?\d+)$`) + +// didNotRun is the marker specdelta puts on a file the treatment could not load. +const didNotRun = "DID NOT RUN on the treatment" + +// module is one rolled-up group: a path prefix and what it holds. +type module struct { + name string + files int + control int + treat int + unloadable int // files the treatment could not load at all + unloadLost int // examples the control passes in those files +} + +// report is what one specdelta file said about itself, so its claim can be +// compared with the rows beneath it. +type report struct { + path string + rows int + control, treat int + saidFiles int + saidControl, saidTreat int + saidDelta int + hasFooter bool +} + +// quiet reports whether this report was produced with specdelta -quiet, which +// prints only the files that MOVED while its footer counts every file walked. +// The two are distinguishable without a flag: a full report has one row per file +// the footer counted. +func (r *report) quiet() bool { return r.hasFooter && r.rows < r.saidFiles } + +// reconcile compares what the rows add up to with what the report claimed, and +// returns the mismatch as a sentence, or "" when they agree. +// +// ⛔ THE INVARIANT IS DIFFERENT IN THE TWO MODES, and using the full-report one +// on a -quiet report would report every quiet report as corrupt. A full report +// must match on all three numbers. A quiet report lists only the files that +// moved -- every file it omits has control == treat and so contributes 0 to the +// delta -- so what must match is the DELTA, which is the number this tool exists +// to attribute. Checking nothing in quiet mode was the other option and it is +// the one that makes a dropped row invisible. +func (r *report) reconcile() string { + if !r.hasFooter { + return fmt.Sprintf("%d row(s) read, and the report has no \"N file(s): control …\" footer to check them against", r.rows) + } + if r.quiet() { + // specdelta's delta is TREATMENT MINUS CONTROL -- negative when rbgo is + // behind. The gap column below is the other way round on purpose (a + // positive "how far behind"), and reconciling with that sign reported + // every correct quiet report as off by twice the gap. + if got := r.treat - r.control; got != r.saidDelta { + return fmt.Sprintf("%d moved row(s) sum to a delta of %+d; the report's own footer says %+d", + r.rows, got, r.saidDelta) + } + return "" + } + if r.rows != r.saidFiles || r.control != r.saidControl || r.treat != r.saidTreat { + return fmt.Sprintf("rows say %d file(s), control %d, treatment %d; the report's own footer says %d, %d, %d", + r.rows, r.control, r.treat, r.saidFiles, r.saidControl, r.saidTreat) + } + return "" +} + +func main() { + depth := flag.Int("depth", 2, "roll up to this many leading path segments (2 = library/date)") + top := flag.Int("top", 0, "print only the N modules with the largest gap (0 = all)") + flag.Parse() + if flag.NArg() == 0 { + fmt.Fprintln(os.Stderr, "specroll: name at least one specdelta report") + flag.Usage() + os.Exit(2) + } + if err := run(os.Stdout, os.Stderr, *depth, *top, flag.Args()); err != nil { + fmt.Fprintln(os.Stderr, "specroll:", err) + os.Exit(exitCode(err)) + } +} + +// codedError carries the exit status the house style distinguishes: 1 is a +// verdict (something disagrees), 2 is a refusal (nothing could be read). +type codedError struct { + code int + msg string +} + +func (e *codedError) Error() string { return e.msg } + +func exitCode(err error) int { + if ce, ok := err.(*codedError); ok { + return ce.code + } + return 2 +} + +func run(stdout, stderr io.Writer, depth, top int, paths []string) error { + if depth < 1 { + return &codedError{2, "-depth must be at least 1"} + } + byName := map[string]*module{} + var reports []*report + + for _, p := range paths { + f, err := os.Open(p) + if err != nil { + return &codedError{2, fmt.Sprintf("open %s: %v", p, err)} + } + rep, err := scan(f, p, depth, byName) + f.Close() + if err != nil { + return err + } + reports = append(reports, rep) + } + + mods := make([]*module, 0, len(byName)) + total := module{name: "TOTAL"} + for _, m := range byName { + mods = append(mods, m) + total.files += m.files + total.control += m.control + total.treat += m.treat + total.unloadable += m.unloadable + total.unloadLost += m.unloadLost + } + if len(mods) == 0 { + // ⛔ Not "no gap": a report this tool cannot parse yields no rows and no + // disagreement either, and an empty table reads exactly like parity. + return &codedError{2, fmt.Sprintf("no specdelta rows in %s; the report format and this parser have drifted apart", + strings.Join(paths, ", "))} + } + + // Largest gap first -- that is the question this answers -- then by name so + // two modules with the same gap keep a stable order between runs. + sort.Slice(mods, func(i, j int) bool { + gi, gj := mods[i].control-mods[i].treat, mods[j].control-mods[j].treat + if gi != gj { + return gi > gj + } + return mods[i].name < mods[j].name + }) + + shown := mods + if top > 0 && top < len(shown) { + shown = shown[:top] + } + + fmt.Fprintf(stdout, "%-34s %6s %8s %7s %7s %s\n", "module", "files", "MRI", "rbgo", "gap", "unloadable (examples)") + for _, m := range shown { + un := "" + if m.unloadable > 0 { + un = fmt.Sprintf("%d (%d)", m.unloadable, m.unloadLost) + } + fmt.Fprintf(stdout, "%-34s %6d %8d %7d %7d %s\n", + m.name, m.files, m.control, m.treat, m.control-m.treat, un) + } + if len(shown) < len(mods) { + fmt.Fprintf(stdout, "… %d more module(s) not shown (-top %d)\n", len(mods)-len(shown), top) + } + fmt.Fprintf(stdout, "%-34s %6d %8d %7d %7d %d (%d)\n", + total.name, total.files, total.control, total.treat, + total.control-total.treat, total.unloadable, total.unloadLost) + + // ⛔ SAY WHAT THE COLUMNS COVER WHEN A QUIET REPORT IS IN THE MIX. A -quiet + // report omits every file where the two engines agreed, so its MRI and rbgo + // columns count the MOVED files only and are not that module's totals. The + // gap column is exact either way -- an omitted file contributes 0 to it -- + // which is why the gap is what this table sorts by and the two count columns + // carry this sentence instead of a footnote nobody reads. + for _, r := range reports { + if r.quiet() { + fmt.Fprintf(stdout, "⛔ some input is a -quiet report: for those modules the MRI and rbgo columns count\n"+ + " only the files that MOVED, not the module's totals. The gap column is exact.\n") + break + } + } + + // The reconciliation, printed whether it passes or not: a control reported + // only on failure is a control nobody can see working. + bad := 0 + for _, r := range reports { + if msg := r.reconcile(); msg != "" { + fmt.Fprintf(stderr, "%s: %s\n", r.path, msg) + bad++ + continue + } + if r.quiet() { + fmt.Fprintf(stdout, "%s: QUIET report — %d moved file(s) of %d walked, delta %+d reconciles with its own footer\n", + r.path, r.rows, r.saidFiles, r.treat-r.control) + continue + } + fmt.Fprintf(stdout, "%s: %d file(s), control %d, treatment %d — rows reconcile with the report's own footer\n", + r.path, r.rows, r.control, r.treat) + } + if bad > 0 { + return &codedError{1, fmt.Sprintf("%d report(s) do not reconcile; the table above is not trustworthy", bad)} + } + return nil +} + +// scan reads one report, adding its rows into byName and returning what the +// report claimed about itself. +// +// ⛔ A -quiet report lists only the files that MOVED, so its rows cannot sum to +// its footer, which counts every file walked. That is not a parse failure and +// must not be reported as one: the footer line is recognised either way, and a +// report whose rows are fewer than its footer's file count is reconciled +// against the moved files alone. See reconcile below. +func scan(r io.Reader, path string, depth int, byName map[string]*module) (*report, error) { + rep := &report{path: path} + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + line := sc.Text() + if m := footerRe.FindStringSubmatch(strings.TrimSpace(line)); m != nil { + rep.hasFooter = true + rep.saidFiles = atoi(m[1]) + rep.saidControl = atoi(m[2]) + rep.saidTreat = atoi(m[3]) + rep.saidDelta = atoi(m[4]) + continue + } + m := rowRe.FindStringSubmatch(line) + if m == nil { + continue + } + name := moduleOf(m[1], depth) + mod := byName[name] + if mod == nil { + mod = &module{name: name} + byName[name] = mod + } + control, treat := atoi(m[2]), atoi(m[3]) + mod.files++ + mod.control += control + mod.treat += treat + if strings.Contains(m[5], didNotRun) { + mod.unloadable++ + mod.unloadLost += control + } + rep.rows++ + rep.control += control + rep.treat += treat + } + if err := sc.Err(); err != nil { + return nil, &codedError{2, fmt.Sprintf("read %s: %v", path, err)} + } + return rep, nil +} + +// moduleOf takes the first depth segments of a spec path, or the whole +// directory when the path is shallower. +func moduleOf(p string, depth int) string { + dir := path.Dir(p) + parts := strings.Split(dir, "/") + if len(parts) > depth { + parts = parts[:depth] + } + return strings.Join(parts, "/") +} + +func atoi(s string) int { + n, err := strconv.Atoi(strings.TrimPrefix(s, "+")) + if err != nil { + return 0 + } + return n +} diff --git a/scripts/conformance/rubyspec/cmd/specroll/main_test.go b/scripts/conformance/rubyspec/cmd/specroll/main_test.go new file mode 100644 index 00000000..f48269c7 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/specroll/main_test.go @@ -0,0 +1,175 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// fullReport is a specdelta report in its DEFAULT shape: one row per file +// walked, footer counting the same files. Every line below is the real format, +// including the indented "WENT RED" lines that sit under a moved row and which +// a column-zero anchor must never read as rows of their own. +const fullReport = `PROVENANCE ⛔ UNPROVENANCED + control /opt/homebrew/opt/ruby@4.0/bin/ruby — no Go build information + treatment /engines/rbgo-parity — built from c604cce1ba0b +MOVE none declared: both engines read the same corpus, so every delta below is the ENGINE + +file control treat delta +library/date/parse_spec.rb 54 22 -32 ⛔ fail+error 0 → 32 + ⛔ WENT RED: error Date#parse | coerces using to_str + ⛔ WENT RED: error Date#parse | parses DD as month day number +library/coverage/result_spec.rb 29 0 -29 DID NOT RUN on the treatment +library/coverage/start_spec.rb 14 0 -14 DID NOT RUN on the treatment +library/date/today_spec.rb 3 3 +0 +library/cgi/escape_spec.rb 4 4 +0 + +5 file(s): control 104 passing, treatment 29 passing, -75 +` + +// quietReport is the same measurement with -quiet: ONLY the moved files are +// printed, so its rows cannot sum to its footer's file count or its passing +// counts -- but the DELTA still must. +const quietReport = `file control treat delta +library/date/parse_spec.rb 54 22 -32 ⛔ fail+error 0 → 32 +library/coverage/result_spec.rb 29 0 -29 DID NOT RUN on the treatment +library/coverage/start_spec.rb 14 0 -14 DID NOT RUN on the treatment + +497 file(s): control 1370 passing, treatment 1295 passing, -75 +` + +func write(t *testing.T, name, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func runRoll(t *testing.T, depth, top int, paths ...string) (string, string, error) { + t.Helper() + var out, errb bytes.Buffer + err := run(&out, &errb, depth, top, paths) + return out.String(), errb.String(), err +} + +// A full report rolls up, reconciles, and separates "behind" from "absent". +func TestAFullReportRollsUpAndReconciles(t *testing.T) { + out, errb, err := runRoll(t, 2, 0, write(t, "full.txt", fullReport)) + if err != nil { + t.Fatalf("run: %v\nstdout: %s\nstderr: %s", err, out, errb) + } + for _, want := range []string{ + // date: 54+3 control, 22+3 treatment, gap 32, nothing unloadable. + "library/date 2 57 25 32", + // coverage: both files unloadable, and the examples they cost named. + "library/coverage 2 43 0 43 2 (43)", + "library/cgi 1 4 4 0", + "TOTAL 5 104 29 75 2 (43)", + "rows reconcile with the report's own footer", + } { + if !strings.Contains(out, want) { + t.Errorf("stdout does not contain %q\n--- got ---\n%s", want, out) + } + } + // ⛔ The indented WENT RED lines must not have become rows: six rows would + // make every column wrong and the footer check is what says so. + if strings.Contains(out, "TOTAL") && strings.Contains(errb, "do not reconcile") { + t.Errorf("the report reconciled in the tool but stderr complains:\n%s", errb) + } +} + +// ⛔ THE CASE THAT MADE THIS TOOL LIE BEFORE IT WAS WRITTEN CORRECTLY. Judged by +// the full-report invariant, every -quiet report is "corrupt": its 3 rows do not +// match a footer counting 497 files. Every report this tool was built to read is +// a quiet one. +func TestAQuietReportIsReconciledByItsDelta(t *testing.T) { + out, errb, err := runRoll(t, 2, 0, write(t, "quiet.txt", quietReport)) + if err != nil { + t.Fatalf("a -quiet report was rejected: %v\nstderr: %s", err, errb) + } + if !strings.Contains(out, "QUIET report — 3 moved file(s) of 497 walked, delta -75 reconciles") { + t.Errorf("stdout does not say the quiet report reconciled:\n%s", out) + } + // And it must say what its columns then cover, or 43 will be read as + // "coverage has 43 examples" rather than "43 of them moved". + if !strings.Contains(out, "count\n only the files that MOVED") { + t.Errorf("a quiet rollup does not state what its count columns cover:\n%s", out) + } +} + +// The same sentence must NOT appear for a full report, where the columns are +// the modules' totals: a caveat printed always is a caveat nobody reads. +func TestAFullRollupDoesNotCarryTheQuietCaveat(t *testing.T) { + out, _, err := runRoll(t, 2, 0, write(t, "full.txt", fullReport)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "only the files that MOVED") { + t.Errorf("a full rollup carries the -quiet caveat:\n%s", out) + } +} + +// A row dropped by a parser that stopped matching must be a REFUSAL, not a +// tidier table. Here the footer claims a delta the rows cannot produce. +func TestRowsThatContradictTheFooterAreRefused(t *testing.T) { + bad := strings.Replace(quietReport, "-75", "-999", 1) + out, errb, err := runRoll(t, 2, 0, write(t, "bad.txt", bad)) + if err == nil { + t.Fatalf("a report whose rows contradict its footer was accepted\nstdout: %s", out) + } + if got := exitCode(err); got != 1 { + t.Errorf("exit status %d, want 1", got) + } + if !strings.Contains(errb, "the report's own footer says -999") { + t.Errorf("stderr does not name the disagreement:\n%s", errb) + } +} + +// A report this parser cannot read yields no rows, and an empty table reads +// exactly like parity. It must refuse. +func TestAnUnparsableReportRefuses(t *testing.T) { + _, _, err := runRoll(t, 2, 0, write(t, "empty.txt", "nothing here that looks like a sweep\n")) + if err == nil { + t.Fatal("an unparsable report was accepted as a clean rollup") + } + if got := exitCode(err); got != 2 { + t.Errorf("exit status %d, want 2 (a refusal, not a verdict)", got) + } +} + +// Several reports roll up together, which is how a corpus split into chunks is +// read back as one answer. +func TestSeveralReportsRollUpTogether(t *testing.T) { + a := write(t, "a.txt", fullReport) + b := write(t, "b.txt", strings.ReplaceAll(fullReport, "library/", "core/")) + out, errb, err := runRoll(t, 1, 0, a, b) + if err != nil { + t.Fatalf("run: %v\nstderr: %s", err, errb) + } + // -depth 1 collapses each tree to its top-level name. + for _, want := range []string{"library", "core", "TOTAL 10 208 58 150"} { + if !strings.Contains(out, want) { + t.Errorf("stdout does not contain %q\n--- got ---\n%s", want, out) + } + } +} + +// -top keeps the table short and says how many it hid: a truncated list that +// does not say it is truncated is the same failure as an unstated population. +func TestTopSaysWhatItHid(t *testing.T) { + out, _, err := runRoll(t, 2, 1, write(t, "full.txt", fullReport)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "more module(s) not shown") { + t.Errorf("a truncated table does not say so:\n%s", out) + } +}