diff --git a/scripts/conformance/rubyspec/cmd/specroll/main.go b/scripts/conformance/rubyspec/cmd/specroll/main.go new file mode 100644 index 00000000..7ccfd340 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/specroll/main.go @@ -0,0 +1,338 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +// Command specroll rolls one or more specdelta reports up to the MODULE. +// +// Why it exists. "What is left to reach parity with MRI?" is a question about +// modules -- Date, coverage, socket -- and specdelta answers it per FILE. One +// run of `library/` produced 497 rows, 189 of them moved; the shape of the debt +// (two Date files carry 53 of the 634 missing examples, five coverage files +// carry 53 more and do not load at all) is not visible by reading 497 lines, and +// a human summarising them by eye is a count done by eye. +// +// ⛔ IT CHECKS ITS OWN ARITHMETIC AGAINST THE REPORT'S. specdelta prints its own +// footer: +// +// 497 file(s): control 1370 passing, treatment 736 passing, -634 +// +// specroll recomputes those three numbers from the rows it parsed and REFUSES +// when they disagree. A rollup is exactly where a dropped row is invisible: the +// table still looks tidy, the modules are still sorted, and the only symptom is +// a total nobody re-derives. The judge and the subject must count alike. +// +// ⛔ IT SEPARATES "BEHIND" FROM "ABSENT". A file the treatment cannot load at +// all is reported by specdelta as DID NOT RUN, and its whole count is missing +// for a reason that is not a failing example -- a missing require, a missing +// class. Those are counted in their own column, because "implement Coverage" +// and "fix 29 Date examples" are different work and summing them hides which. +// +// ⛔ IT IS NOT A SECOND MEASUREMENT. It reads what specdelta measured and says +// nothing specdelta did not; a module missing from the reports is missing here. +// The population is the reports named on the command line, and the footer says +// which files it read, so a rollup of half a corpus cannot be read as a whole. +// +// Style note: stdlib `flag`, like its four neighbours in this directory +// (specdelta, ratchet, phantom, oracle). These are repository scripts, not +// released CLIs, and a cobra dependency for one of five would be the odd one +// out in the directory that matters most for reading them side by side. +// +// Usage: +// +// specroll [-depth N] [-top N] ... +// +// Exit status: 0 the rollup printed and reconciled, 1 a report's own totals +// disagree with the rows it contains, 2 nothing could be read. +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "path" + "regexp" + "sort" + "strconv" + "strings" +) + +// A specdelta per-file row: +// +// library/date/parse_spec.rb 54 22 -32 ⛔ fail+error 0 → 32 +// +// Anchored at column zero so the indented "⛔ WENT RED:" lines beneath a row -- +// which also contain digits and arrows -- can never be read as rows. +var rowRe = regexp.MustCompile(`^(\S+\.rb)\s+(\d+)\s+(\d+)\s+([+-]\d+)\s*(.*)$`) + +// specdelta's footer, the control this tool judges itself against. +var footerRe = regexp.MustCompile(`^(\d+) file\(s\): control (\d+) passing, treatment (\d+) passing, ([+-]?\d+)$`) + +// didNotRun is the marker specdelta puts on a file the treatment could not load. +const didNotRun = "DID NOT RUN on the treatment" + +// module is one rolled-up group: a path prefix and what it holds. +type module struct { + name string + files int + control int + treat int + unloadable int // files the treatment could not load at all + unloadLost int // examples the control passes in those files +} + +// report is what one specdelta file said about itself, so its claim can be +// compared with the rows beneath it. +type report struct { + path string + rows int + control, treat int + saidFiles int + saidControl, saidTreat int + saidDelta int + hasFooter bool +} + +// quiet reports whether this report was produced with specdelta -quiet, which +// prints only the files that MOVED while its footer counts every file walked. +// The two are distinguishable without a flag: a full report has one row per file +// the footer counted. +func (r *report) quiet() bool { return r.hasFooter && r.rows < r.saidFiles } + +// reconcile compares what the rows add up to with what the report claimed, and +// returns the mismatch as a sentence, or "" when they agree. +// +// ⛔ THE INVARIANT IS DIFFERENT IN THE TWO MODES, and using the full-report one +// on a -quiet report would report every quiet report as corrupt. A full report +// must match on all three numbers. A quiet report lists only the files that +// moved -- every file it omits has control == treat and so contributes 0 to the +// delta -- so what must match is the DELTA, which is the number this tool exists +// to attribute. Checking nothing in quiet mode was the other option and it is +// the one that makes a dropped row invisible. +func (r *report) reconcile() string { + if !r.hasFooter { + return fmt.Sprintf("%d row(s) read, and the report has no \"N file(s): control …\" footer to check them against", r.rows) + } + if r.quiet() { + // specdelta's delta is TREATMENT MINUS CONTROL -- negative when rbgo is + // behind. The gap column below is the other way round on purpose (a + // positive "how far behind"), and reconciling with that sign reported + // every correct quiet report as off by twice the gap. + if got := r.treat - r.control; got != r.saidDelta { + return fmt.Sprintf("%d moved row(s) sum to a delta of %+d; the report's own footer says %+d", + r.rows, got, r.saidDelta) + } + return "" + } + if r.rows != r.saidFiles || r.control != r.saidControl || r.treat != r.saidTreat { + return fmt.Sprintf("rows say %d file(s), control %d, treatment %d; the report's own footer says %d, %d, %d", + r.rows, r.control, r.treat, r.saidFiles, r.saidControl, r.saidTreat) + } + return "" +} + +func main() { + depth := flag.Int("depth", 2, "roll up to this many leading path segments (2 = library/date)") + top := flag.Int("top", 0, "print only the N modules with the largest gap (0 = all)") + flag.Parse() + if flag.NArg() == 0 { + fmt.Fprintln(os.Stderr, "specroll: name at least one specdelta report") + flag.Usage() + os.Exit(2) + } + if err := run(os.Stdout, os.Stderr, *depth, *top, flag.Args()); err != nil { + fmt.Fprintln(os.Stderr, "specroll:", err) + os.Exit(exitCode(err)) + } +} + +// codedError carries the exit status the house style distinguishes: 1 is a +// verdict (something disagrees), 2 is a refusal (nothing could be read). +type codedError struct { + code int + msg string +} + +func (e *codedError) Error() string { return e.msg } + +func exitCode(err error) int { + if ce, ok := err.(*codedError); ok { + return ce.code + } + return 2 +} + +func run(stdout, stderr io.Writer, depth, top int, paths []string) error { + if depth < 1 { + return &codedError{2, "-depth must be at least 1"} + } + byName := map[string]*module{} + var reports []*report + + for _, p := range paths { + f, err := os.Open(p) + if err != nil { + return &codedError{2, fmt.Sprintf("open %s: %v", p, err)} + } + rep, err := scan(f, p, depth, byName) + f.Close() + if err != nil { + return err + } + reports = append(reports, rep) + } + + mods := make([]*module, 0, len(byName)) + total := module{name: "TOTAL"} + for _, m := range byName { + mods = append(mods, m) + total.files += m.files + total.control += m.control + total.treat += m.treat + total.unloadable += m.unloadable + total.unloadLost += m.unloadLost + } + if len(mods) == 0 { + // ⛔ Not "no gap": a report this tool cannot parse yields no rows and no + // disagreement either, and an empty table reads exactly like parity. + return &codedError{2, fmt.Sprintf("no specdelta rows in %s; the report format and this parser have drifted apart", + strings.Join(paths, ", "))} + } + + // Largest gap first -- that is the question this answers -- then by name so + // two modules with the same gap keep a stable order between runs. + sort.Slice(mods, func(i, j int) bool { + gi, gj := mods[i].control-mods[i].treat, mods[j].control-mods[j].treat + if gi != gj { + return gi > gj + } + return mods[i].name < mods[j].name + }) + + shown := mods + if top > 0 && top < len(shown) { + shown = shown[:top] + } + + fmt.Fprintf(stdout, "%-34s %6s %8s %7s %7s %s\n", "module", "files", "MRI", "rbgo", "gap", "unloadable (examples)") + for _, m := range shown { + un := "" + if m.unloadable > 0 { + un = fmt.Sprintf("%d (%d)", m.unloadable, m.unloadLost) + } + fmt.Fprintf(stdout, "%-34s %6d %8d %7d %7d %s\n", + m.name, m.files, m.control, m.treat, m.control-m.treat, un) + } + if len(shown) < len(mods) { + fmt.Fprintf(stdout, "… %d more module(s) not shown (-top %d)\n", len(mods)-len(shown), top) + } + fmt.Fprintf(stdout, "%-34s %6d %8d %7d %7d %d (%d)\n", + total.name, total.files, total.control, total.treat, + total.control-total.treat, total.unloadable, total.unloadLost) + + // ⛔ SAY WHAT THE COLUMNS COVER WHEN A QUIET REPORT IS IN THE MIX. A -quiet + // report omits every file where the two engines agreed, so its MRI and rbgo + // columns count the MOVED files only and are not that module's totals. The + // gap column is exact either way -- an omitted file contributes 0 to it -- + // which is why the gap is what this table sorts by and the two count columns + // carry this sentence instead of a footnote nobody reads. + for _, r := range reports { + if r.quiet() { + fmt.Fprintf(stdout, "⛔ some input is a -quiet report: for those modules the MRI and rbgo columns count\n"+ + " only the files that MOVED, not the module's totals. The gap column is exact.\n") + break + } + } + + // The reconciliation, printed whether it passes or not: a control reported + // only on failure is a control nobody can see working. + bad := 0 + for _, r := range reports { + if msg := r.reconcile(); msg != "" { + fmt.Fprintf(stderr, "%s: %s\n", r.path, msg) + bad++ + continue + } + if r.quiet() { + fmt.Fprintf(stdout, "%s: QUIET report — %d moved file(s) of %d walked, delta %+d reconciles with its own footer\n", + r.path, r.rows, r.saidFiles, r.treat-r.control) + continue + } + fmt.Fprintf(stdout, "%s: %d file(s), control %d, treatment %d — rows reconcile with the report's own footer\n", + r.path, r.rows, r.control, r.treat) + } + if bad > 0 { + return &codedError{1, fmt.Sprintf("%d report(s) do not reconcile; the table above is not trustworthy", bad)} + } + return nil +} + +// scan reads one report, adding its rows into byName and returning what the +// report claimed about itself. +// +// ⛔ A -quiet report lists only the files that MOVED, so its rows cannot sum to +// its footer, which counts every file walked. That is not a parse failure and +// must not be reported as one: the footer line is recognised either way, and a +// report whose rows are fewer than its footer's file count is reconciled +// against the moved files alone. See reconcile below. +func scan(r io.Reader, path string, depth int, byName map[string]*module) (*report, error) { + rep := &report{path: path} + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + line := sc.Text() + if m := footerRe.FindStringSubmatch(strings.TrimSpace(line)); m != nil { + rep.hasFooter = true + rep.saidFiles = atoi(m[1]) + rep.saidControl = atoi(m[2]) + rep.saidTreat = atoi(m[3]) + rep.saidDelta = atoi(m[4]) + continue + } + m := rowRe.FindStringSubmatch(line) + if m == nil { + continue + } + name := moduleOf(m[1], depth) + mod := byName[name] + if mod == nil { + mod = &module{name: name} + byName[name] = mod + } + control, treat := atoi(m[2]), atoi(m[3]) + mod.files++ + mod.control += control + mod.treat += treat + if strings.Contains(m[5], didNotRun) { + mod.unloadable++ + mod.unloadLost += control + } + rep.rows++ + rep.control += control + rep.treat += treat + } + if err := sc.Err(); err != nil { + return nil, &codedError{2, fmt.Sprintf("read %s: %v", path, err)} + } + return rep, nil +} + +// moduleOf takes the first depth segments of a spec path, or the whole +// directory when the path is shallower. +func moduleOf(p string, depth int) string { + dir := path.Dir(p) + parts := strings.Split(dir, "/") + if len(parts) > depth { + parts = parts[:depth] + } + return strings.Join(parts, "/") +} + +func atoi(s string) int { + n, err := strconv.Atoi(strings.TrimPrefix(s, "+")) + if err != nil { + return 0 + } + return n +} diff --git a/scripts/conformance/rubyspec/cmd/specroll/main_test.go b/scripts/conformance/rubyspec/cmd/specroll/main_test.go new file mode 100644 index 00000000..f48269c7 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/specroll/main_test.go @@ -0,0 +1,175 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// fullReport is a specdelta report in its DEFAULT shape: one row per file +// walked, footer counting the same files. Every line below is the real format, +// including the indented "WENT RED" lines that sit under a moved row and which +// a column-zero anchor must never read as rows of their own. +const fullReport = `PROVENANCE ⛔ UNPROVENANCED + control /opt/homebrew/opt/ruby@4.0/bin/ruby — no Go build information + treatment /engines/rbgo-parity — built from c604cce1ba0b +MOVE none declared: both engines read the same corpus, so every delta below is the ENGINE + +file control treat delta +library/date/parse_spec.rb 54 22 -32 ⛔ fail+error 0 → 32 + ⛔ WENT RED: error Date#parse | coerces using to_str + ⛔ WENT RED: error Date#parse | parses DD as month day number +library/coverage/result_spec.rb 29 0 -29 DID NOT RUN on the treatment +library/coverage/start_spec.rb 14 0 -14 DID NOT RUN on the treatment +library/date/today_spec.rb 3 3 +0 +library/cgi/escape_spec.rb 4 4 +0 + +5 file(s): control 104 passing, treatment 29 passing, -75 +` + +// quietReport is the same measurement with -quiet: ONLY the moved files are +// printed, so its rows cannot sum to its footer's file count or its passing +// counts -- but the DELTA still must. +const quietReport = `file control treat delta +library/date/parse_spec.rb 54 22 -32 ⛔ fail+error 0 → 32 +library/coverage/result_spec.rb 29 0 -29 DID NOT RUN on the treatment +library/coverage/start_spec.rb 14 0 -14 DID NOT RUN on the treatment + +497 file(s): control 1370 passing, treatment 1295 passing, -75 +` + +func write(t *testing.T, name, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func runRoll(t *testing.T, depth, top int, paths ...string) (string, string, error) { + t.Helper() + var out, errb bytes.Buffer + err := run(&out, &errb, depth, top, paths) + return out.String(), errb.String(), err +} + +// A full report rolls up, reconciles, and separates "behind" from "absent". +func TestAFullReportRollsUpAndReconciles(t *testing.T) { + out, errb, err := runRoll(t, 2, 0, write(t, "full.txt", fullReport)) + if err != nil { + t.Fatalf("run: %v\nstdout: %s\nstderr: %s", err, out, errb) + } + for _, want := range []string{ + // date: 54+3 control, 22+3 treatment, gap 32, nothing unloadable. + "library/date 2 57 25 32", + // coverage: both files unloadable, and the examples they cost named. + "library/coverage 2 43 0 43 2 (43)", + "library/cgi 1 4 4 0", + "TOTAL 5 104 29 75 2 (43)", + "rows reconcile with the report's own footer", + } { + if !strings.Contains(out, want) { + t.Errorf("stdout does not contain %q\n--- got ---\n%s", want, out) + } + } + // ⛔ The indented WENT RED lines must not have become rows: six rows would + // make every column wrong and the footer check is what says so. + if strings.Contains(out, "TOTAL") && strings.Contains(errb, "do not reconcile") { + t.Errorf("the report reconciled in the tool but stderr complains:\n%s", errb) + } +} + +// ⛔ THE CASE THAT MADE THIS TOOL LIE BEFORE IT WAS WRITTEN CORRECTLY. Judged by +// the full-report invariant, every -quiet report is "corrupt": its 3 rows do not +// match a footer counting 497 files. Every report this tool was built to read is +// a quiet one. +func TestAQuietReportIsReconciledByItsDelta(t *testing.T) { + out, errb, err := runRoll(t, 2, 0, write(t, "quiet.txt", quietReport)) + if err != nil { + t.Fatalf("a -quiet report was rejected: %v\nstderr: %s", err, errb) + } + if !strings.Contains(out, "QUIET report — 3 moved file(s) of 497 walked, delta -75 reconciles") { + t.Errorf("stdout does not say the quiet report reconciled:\n%s", out) + } + // And it must say what its columns then cover, or 43 will be read as + // "coverage has 43 examples" rather than "43 of them moved". + if !strings.Contains(out, "count\n only the files that MOVED") { + t.Errorf("a quiet rollup does not state what its count columns cover:\n%s", out) + } +} + +// The same sentence must NOT appear for a full report, where the columns are +// the modules' totals: a caveat printed always is a caveat nobody reads. +func TestAFullRollupDoesNotCarryTheQuietCaveat(t *testing.T) { + out, _, err := runRoll(t, 2, 0, write(t, "full.txt", fullReport)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "only the files that MOVED") { + t.Errorf("a full rollup carries the -quiet caveat:\n%s", out) + } +} + +// A row dropped by a parser that stopped matching must be a REFUSAL, not a +// tidier table. Here the footer claims a delta the rows cannot produce. +func TestRowsThatContradictTheFooterAreRefused(t *testing.T) { + bad := strings.Replace(quietReport, "-75", "-999", 1) + out, errb, err := runRoll(t, 2, 0, write(t, "bad.txt", bad)) + if err == nil { + t.Fatalf("a report whose rows contradict its footer was accepted\nstdout: %s", out) + } + if got := exitCode(err); got != 1 { + t.Errorf("exit status %d, want 1", got) + } + if !strings.Contains(errb, "the report's own footer says -999") { + t.Errorf("stderr does not name the disagreement:\n%s", errb) + } +} + +// A report this parser cannot read yields no rows, and an empty table reads +// exactly like parity. It must refuse. +func TestAnUnparsableReportRefuses(t *testing.T) { + _, _, err := runRoll(t, 2, 0, write(t, "empty.txt", "nothing here that looks like a sweep\n")) + if err == nil { + t.Fatal("an unparsable report was accepted as a clean rollup") + } + if got := exitCode(err); got != 2 { + t.Errorf("exit status %d, want 2 (a refusal, not a verdict)", got) + } +} + +// Several reports roll up together, which is how a corpus split into chunks is +// read back as one answer. +func TestSeveralReportsRollUpTogether(t *testing.T) { + a := write(t, "a.txt", fullReport) + b := write(t, "b.txt", strings.ReplaceAll(fullReport, "library/", "core/")) + out, errb, err := runRoll(t, 1, 0, a, b) + if err != nil { + t.Fatalf("run: %v\nstderr: %s", err, errb) + } + // -depth 1 collapses each tree to its top-level name. + for _, want := range []string{"library", "core", "TOTAL 10 208 58 150"} { + if !strings.Contains(out, want) { + t.Errorf("stdout does not contain %q\n--- got ---\n%s", want, out) + } + } +} + +// -top keeps the table short and says how many it hid: a truncated list that +// does not say it is truncated is the same failure as an unstated population. +func TestTopSaysWhatItHid(t *testing.T) { + out, _, err := runRoll(t, 2, 1, write(t, "full.txt", fullReport)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "more module(s) not shown") { + t.Errorf("a truncated table does not say so:\n%s", out) + } +}