diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main.go b/scripts/conformance/rubyspec/cmd/specdelta/main.go index 228b39f1..88422669 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main.go @@ -105,6 +105,40 @@ type result struct { // with one `skip` detail, entirely well-formed. Reading the shim rather than // assuming its shape is what produced the table above; the line // `hookerror` has no counter is a defect in the harness, filed separately. +// outcomeNote is the note printed beside a row when one of the two engines did +// not produce an ordinary result: it was stopped by the clock, or it produced +// nothing at all. +// +// ⛔ THE KILL IS TESTED FIRST, AND THE ORDER IS THE WHOLE POINT. A killed engine +// almost never prints a RESULT line, so `ran` is false and the DID-NOT-RUN arms +// would claim every killed row -- reporting a file the CLOCK stopped as a file +// the engine could not LOAD. Those are opposite findings: one says the timeout +// was too short or the engine hangs, the other says the feature is +// unimplemented, and they call for opposite work. +// +// It was worse than an ordering bug before this: run() returned `parse(...)`, +// a fresh result, so `killed` was false on every row ever printed and this +// branch was unreachable. The documented control for a whole report -- "grep +// for KILLED; if it is 0 the table is sound" -- could only ever answer 0, and +// every sweep looked clean by construction. +func outcomeNote(a, b result) string { + switch { + case a.killed && b.killed: + return " KILLED on timeout, both engines" + case a.killed: + return " KILLED on timeout, the control" + case b.killed: + return " KILLED on timeout, the treatment" + case !a.ran && !b.ran: + return " DID NOT RUN on either" + case !a.ran: + return " DID NOT RUN on the control" + case !b.ran: + return " DID NOT RUN on the treatment" + } + return "" +} + func (r result) tallyDisagrees() bool { return r.ran && r.details < r.fail+r.errs } @@ -259,17 +293,7 @@ func main1(args []string, out, errOut io.Writer) int { } totalA += r.a.pass totalB += r.b.pass - note := "" - switch { - case !r.a.ran && !r.b.ran: - note = " DID NOT RUN on either" - case !r.a.ran: - note = " DID NOT RUN on the control" - case !r.b.ran: - note = " DID NOT RUN on the treatment" - case r.a.killed || r.b.killed: - note = " KILLED on timeout" - } + note := outcomeNote(r.a, r.b) if r.unstable { note += " ⛔ UNSTABLE: an engine disagreed with itself, delta excluded" } @@ -807,7 +831,48 @@ func run(engine, root, file string, timeout time.Duration) (result, error) { <-done r.killed = true } - return parse(out.String()) + // ⛔ CARRY THE KILL ACROSS THE PARSE. This used to `return parse(...)`, which + // builds a FRESH result and threw `killed` away on the floor -- so `killed` + // was false on every row ever printed and the "KILLED on timeout" note was + // dead code. A killed engine usually prints no RESULT line either, so it fell + // through to `ran == false` and the report said "DID NOT RUN on the + // treatment": a HANG was published as an ABSENCE, which is a statement about + // conformance rather than about the clock, and the two call for opposite work. + // + // It also broke the control for the report as a whole. "grep KILLED; if 0 the + // table is sound" is the documented way to tell whether the machine's load + // decided a cell -- and that grep could only ever answer 0. + parsed, err := parse(printable(out.String())) + parsed.killed = r.killed + return parsed, err +} + +// printable strips the control bytes a spec's own output can carry into this +// report. ruby/spec files print raw binary on purpose -- zlib round-trips, +// net-http bodies, String#unpack -- and the shim's RBGO_DETAIL line quotes the +// example's description and message verbatim. +// +// ⛔ MEASURED 2026-11-10. One library/ report came out holding NUL bytes, and +// `file` called it "RAGE Package Format (RPF)". Every grep over it then +// returned NOTHING AT ALL -- not zero matches, no output and exit 1, because +// grep skips a file it decides is binary. Three separate checks of that report +// ("is there a KILLED row?", "which files did not load?") were read as clean +// when the search had never looked inside. A report that ordinary tools refuse +// to read is a report whose silence cannot be told from a clean one. +// +// run.sh has stripped NULs since it was written, for the same reason and one +// layer further out (`tr -d '\000'`); this is that guard where the Go half can +// also rely on it. Tab and newline are kept, since the format is built on them. +func printable(s string) string { + return strings.Map(func(r rune) rune { + if r == '\t' || r == '\n' { + return r + } + if r < 0x20 || r == 0x7f { + return -1 + } + return r + }, s) } // parse reads the shim's report. It REFUSES a RESULT line it cannot read rather diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go index 051abfc2..92d379fb 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go @@ -11,6 +11,7 @@ import ( "runtime" "strings" "testing" + "time" ) // parse is where a silent wrong answer would live, so it is tested first and @@ -844,3 +845,113 @@ func fakeEngine(t *testing.T, dir, name, body string) string { } return p } + +// ⛔ THE CONTROL THAT COULD NOT FIRE. run() used to `return parse(...)`, which +// builds a fresh result, so the `killed` it had just set was discarded. Every +// row printed carried killed == false, the "KILLED on timeout" note was +// unreachable, and the documented way to check a whole report -- grep it for +// KILLED, and trust the table when the count is 0 -- answered 0 by +// construction. Measured 2026-11-09: three sweeps of the ruby/spec corpus were +// declared sound on that grep. +func TestATimeoutIsCarriedOutOfRun(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses /bin/sleep as a deliberately slow engine") + } + // specdelta runs `engine `, so /bin/sleep with "30" as the "spec file" + // is an engine that is guaranteed to outlive any timeout this test sets. + r, err := run("/bin/sleep", t.TempDir(), "30", 150*time.Millisecond) + if err != nil { + t.Fatalf("run: %v", err) + } + if !r.killed { + t.Error("an engine stopped by the timeout came back with killed == false; " + + "the report cannot then tell a hang from a file the engine could not load") + } + if r.ran { + t.Error("a killed engine printed no RESULT line, so ran must be false") + } +} + +// An engine that finishes inside the timeout must NOT be marked killed, or the +// note would fire on every row and the control would be useless the other way. +func TestAnEngineThatFinishesIsNotMarkedKilled(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses /bin/sleep as a deliberately quick engine") + } + r, err := run("/bin/sleep", t.TempDir(), "0", 10*time.Second) + if err != nil { + t.Fatalf("run: %v", err) + } + if r.killed { + t.Error("an engine that exited on its own was reported as killed") + } +} + +// ⛔ The ORDER of the note, which is what turned a hang into an absence. A +// killed engine prints no RESULT line, so ran is false too; if DID NOT RUN is +// tested first it claims every killed row. +func TestAKillIsReportedBeforeAnAbsence(t *testing.T) { + ranOK := result{ran: true, pass: 3} + killedSilent := result{killed: true} // what a real timeout looks like + for _, tc := range []struct { + name string + a, b result + want string + }{ + {"the treatment was stopped by the clock", ranOK, killedSilent, " KILLED on timeout, the treatment"}, + {"the control was stopped by the clock", killedSilent, ranOK, " KILLED on timeout, the control"}, + {"both were", killedSilent, killedSilent, " KILLED on timeout, both engines"}, + {"the treatment genuinely produced nothing", ranOK, result{}, " DID NOT RUN on the treatment"}, + {"the control genuinely produced nothing", result{}, ranOK, " DID NOT RUN on the control"}, + {"neither produced anything", result{}, result{}, " DID NOT RUN on either"}, + {"both ran", ranOK, ranOK, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := outcomeNote(tc.a, tc.b); got != tc.want { + t.Errorf("outcomeNote = %q, want %q", got, tc.want) + } + }) + } +} + +// ⛔ A REPORT NO TOOL WILL READ. ruby/spec prints raw binary on purpose (zlib +// round-trips, net-http bodies, String#unpack), and the shim quotes an +// example's description verbatim into RBGO_DETAIL. Measured 2026-11-10: one +// library/ report came out holding NUL bytes, `file` called it "RAGE Package +// Format (RPF)", and every grep over it returned NOTHING -- no output, exit 1, +// because grep skips what it decides is binary. Three checks of that report +// were read as clean when the search had never looked inside it. +func TestControlBytesFromASpecDoNotReachTheReport(t *testing.T) { + const dirty = "RBGO_DETAIL\tfail\tZlib\tround\x00trips \x01binary\x7f\tmsg\n" + + "RBGO_RESULT pass=1 fail=1 error=0 skip=0\n" + r, err := parse(printable(dirty)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !r.ran || r.pass != 1 { + t.Fatalf("the surrounding report must still parse: ran=%v pass=%d", r.ran, r.pass) + } + for label := range r.labels { + if strings.ContainsAny(label, "\x00\x01\x7f") { + t.Errorf("a control byte reached a label: %q", label) + } + } + // The text either side of the stripped byte must survive: dropping the whole + // description would lose the finding instead of the byte. + found := false + for label := range r.labels { + if strings.Contains(label, "roundtrips binary") { + found = true + } + } + if !found { + t.Errorf("the description did not survive the strip: %v", r.labels) + } +} + +// Tab and newline ARE the format and must not be stripped with the rest. +func TestPrintableKeepsTheFormatsOwnSeparators(t *testing.T) { + if got := printable("a\tb\nc"); got != "a\tb\nc" { + t.Errorf("printable mangled the separators: %q", got) + } +}