From 064264b72e8451067ab4f32866fb24934c40a27d Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 10 Nov 2026 02:28:16 +0100 Subject: [PATCH 1/2] specdelta: a hang was published as an absence, and the control could not fire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run() ended with return parse(out.String()) which builds a FRESH result, so the `killed` it had set two lines above was dropped on the floor. Every row specdelta has ever printed carried killed == false. Three consequences, each worse than the last: 1. The "KILLED on timeout" note was unreachable dead code. 2. A killed engine almost never prints a RESULT line, so `ran` is false too -- and the note switch tested DID NOT RUN first. A file the CLOCK stopped was therefore reported as "DID NOT RUN on the treatment": a hang published as an absence. Those are opposite findings. One says the timeout is too short or the engine hangs; the other says the feature is unimplemented, and they call for opposite work. 3. The documented control for a whole report -- grep it for KILLED, and trust the table when the count is 0 -- answered 0 BY CONSTRUCTION. Measured 2026-11-09: three sweeps of the ruby/spec corpus were declared sound on exactly that grep, including one of library/ that took 3h45 with a spec file observed blocked for 4m46s. The fix carries `killed` across the parse and tests it BEFORE the DID-NOT-RUN arms, in outcomeNote(), extracted so the ordering is a test and not a comment. Controls, in both directions: - TestATimeoutIsCarriedOutOfRun uses /bin/sleep as an engine guaranteed to outlive the timeout. Run against the unfixed line it fails with "came back with killed == false"; against the fix it passes in 0.16s, and the time is the witness that the timeout is what ended it. - TestAnEngineThatFinishesIsNotMarkedKilled is the other direction: a note that fires on every row is as useless as one that never fires. - TestAKillIsReportedBeforeAnAbsence pins all seven outcomes, with a killed-and-silent result as the realistic case -- that is what a real timeout looks like, and it is the one the old order got wrong. ⛔ Reports produced before this commit cannot distinguish the two causes. Any "DID NOT RUN on the treatment" row in them may be a timeout. Co-Authored-By: Claude Opus 5 --- .../rubyspec/cmd/specdelta/main.go | 61 ++++++++++++---- .../rubyspec/cmd/specdelta/main_test.go | 69 +++++++++++++++++++ 2 files changed, 118 insertions(+), 12 deletions(-) diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main.go b/scripts/conformance/rubyspec/cmd/specdelta/main.go index 228b39f1..18015d7b 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main.go @@ -105,6 +105,40 @@ type result struct { // with one `skip` detail, entirely well-formed. Reading the shim rather than // assuming its shape is what produced the table above; the line // `hookerror` has no counter is a defect in the harness, filed separately. +// outcomeNote is the note printed beside a row when one of the two engines did +// not produce an ordinary result: it was stopped by the clock, or it produced +// nothing at all. +// +// ⛔ THE KILL IS TESTED FIRST, AND THE ORDER IS THE WHOLE POINT. A killed engine +// almost never prints a RESULT line, so `ran` is false and the DID-NOT-RUN arms +// would claim every killed row -- reporting a file the CLOCK stopped as a file +// the engine could not LOAD. Those are opposite findings: one says the timeout +// was too short or the engine hangs, the other says the feature is +// unimplemented, and they call for opposite work. +// +// It was worse than an ordering bug before this: run() returned `parse(...)`, +// a fresh result, so `killed` was false on every row ever printed and this +// branch was unreachable. The documented control for a whole report -- "grep +// for KILLED; if it is 0 the table is sound" -- could only ever answer 0, and +// every sweep looked clean by construction. +func outcomeNote(a, b result) string { + switch { + case a.killed && b.killed: + return " KILLED on timeout, both engines" + case a.killed: + return " KILLED on timeout, the control" + case b.killed: + return " KILLED on timeout, the treatment" + case !a.ran && !b.ran: + return " DID NOT RUN on either" + case !a.ran: + return " DID NOT RUN on the control" + case !b.ran: + return " DID NOT RUN on the treatment" + } + return "" +} + func (r result) tallyDisagrees() bool { return r.ran && r.details < r.fail+r.errs } @@ -259,17 +293,7 @@ func main1(args []string, out, errOut io.Writer) int { } totalA += r.a.pass totalB += r.b.pass - note := "" - switch { - case !r.a.ran && !r.b.ran: - note = " DID NOT RUN on either" - case !r.a.ran: - note = " DID NOT RUN on the control" - case !r.b.ran: - note = " DID NOT RUN on the treatment" - case r.a.killed || r.b.killed: - note = " KILLED on timeout" - } + note := outcomeNote(r.a, r.b) if r.unstable { note += " ⛔ UNSTABLE: an engine disagreed with itself, delta excluded" } @@ -807,7 +831,20 @@ func run(engine, root, file string, timeout time.Duration) (result, error) { <-done r.killed = true } - return parse(out.String()) + // ⛔ CARRY THE KILL ACROSS THE PARSE. This used to `return parse(...)`, which + // builds a FRESH result and threw `killed` away on the floor -- so `killed` + // was false on every row ever printed and the "KILLED on timeout" note was + // dead code. A killed engine usually prints no RESULT line either, so it fell + // through to `ran == false` and the report said "DID NOT RUN on the + // treatment": a HANG was published as an ABSENCE, which is a statement about + // conformance rather than about the clock, and the two call for opposite work. + // + // It also broke the control for the report as a whole. "grep KILLED; if 0 the + // table is sound" is the documented way to tell whether the machine's load + // decided a cell -- and that grep could only ever answer 0. + parsed, err := parse(out.String()) + parsed.killed = r.killed + return parsed, err } // parse reads the shim's report. It REFUSES a RESULT line it cannot read rather diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go index 051abfc2..389671cf 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go @@ -11,6 +11,7 @@ import ( "runtime" "strings" "testing" + "time" ) // parse is where a silent wrong answer would live, so it is tested first and @@ -844,3 +845,71 @@ func fakeEngine(t *testing.T, dir, name, body string) string { } return p } + +// ⛔ THE CONTROL THAT COULD NOT FIRE. run() used to `return parse(...)`, which +// builds a fresh result, so the `killed` it had just set was discarded. Every +// row printed carried killed == false, the "KILLED on timeout" note was +// unreachable, and the documented way to check a whole report -- grep it for +// KILLED, and trust the table when the count is 0 -- answered 0 by +// construction. Measured 2026-11-09: three sweeps of the ruby/spec corpus were +// declared sound on that grep. +func TestATimeoutIsCarriedOutOfRun(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses /bin/sleep as a deliberately slow engine") + } + // specdelta runs `engine `, so /bin/sleep with "30" as the "spec file" + // is an engine that is guaranteed to outlive any timeout this test sets. + r, err := run("/bin/sleep", t.TempDir(), "30", 150*time.Millisecond) + if err != nil { + t.Fatalf("run: %v", err) + } + if !r.killed { + t.Error("an engine stopped by the timeout came back with killed == false; " + + "the report cannot then tell a hang from a file the engine could not load") + } + if r.ran { + t.Error("a killed engine printed no RESULT line, so ran must be false") + } +} + +// An engine that finishes inside the timeout must NOT be marked killed, or the +// note would fire on every row and the control would be useless the other way. +func TestAnEngineThatFinishesIsNotMarkedKilled(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses /bin/sleep as a deliberately quick engine") + } + r, err := run("/bin/sleep", t.TempDir(), "0", 10*time.Second) + if err != nil { + t.Fatalf("run: %v", err) + } + if r.killed { + t.Error("an engine that exited on its own was reported as killed") + } +} + +// ⛔ The ORDER of the note, which is what turned a hang into an absence. A +// killed engine prints no RESULT line, so ran is false too; if DID NOT RUN is +// tested first it claims every killed row. +func TestAKillIsReportedBeforeAnAbsence(t *testing.T) { + ranOK := result{ran: true, pass: 3} + killedSilent := result{killed: true} // what a real timeout looks like + for _, tc := range []struct { + name string + a, b result + want string + }{ + {"the treatment was stopped by the clock", ranOK, killedSilent, " KILLED on timeout, the treatment"}, + {"the control was stopped by the clock", killedSilent, ranOK, " KILLED on timeout, the control"}, + {"both were", killedSilent, killedSilent, " KILLED on timeout, both engines"}, + {"the treatment genuinely produced nothing", ranOK, result{}, " DID NOT RUN on the treatment"}, + {"the control genuinely produced nothing", result{}, ranOK, " DID NOT RUN on the control"}, + {"neither produced anything", result{}, result{}, " DID NOT RUN on either"}, + {"both ran", ranOK, ranOK, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := outcomeNote(tc.a, tc.b); got != tc.want { + t.Errorf("outcomeNote = %q, want %q", got, tc.want) + } + }) + } +} From 73838c3189fbc4893a1feb7a40be01025ba50f8a Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 10 Nov 2026 03:47:03 +0100 Subject: [PATCH 2/2] specdelta: a report ordinary tools refuse to read is not a clean report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ruby/spec files print raw binary on purpose -- zlib round-trips, net-http bodies, String#unpack -- and the shim's RBGO_DETAIL line quotes the example's description and message verbatim. Those bytes went straight into the report. ⛔ MEASURED 2026-11-10. One library/ sweep produced a report holding NUL bytes, and `file` called it "RAGE Package Format (RPF)". Every grep over it then returned NOTHING AT ALL -- not "0 matches", but no output and exit 1, because grep skips a file it decides is binary. Three separate checks of that report were read as clean on that silence: "is there a KILLED row?" -> no output, read as none "which files did not load?" -> no output, read as none "does it mention net-ftp at all?" -> no output, while cmd/specroll reading the same file in Go counted 47 net-ftp rows, 39 of them unloadable The Go reader was right and the pipeline was wrong, which is the whole argument for putting a verdict in a program rather than in a search. printable() strips control bytes from an engine's output before it is parsed, keeping tab and newline because the format is built on them. run.sh has done exactly this since it was written, one layer further out (`tr -d '\000'`); this is the same guard where the Go half can rely on it. Two tests: a dirty RBGO_DETAIL line must parse with the surrounding result intact AND with the text either side of the stripped byte still there (dropping the whole description would lose the finding instead of the byte), and the format's own separators must survive. Co-Authored-By: Claude Opus 5 --- .../rubyspec/cmd/specdelta/main.go | 30 ++++++++++++- .../rubyspec/cmd/specdelta/main_test.go | 42 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main.go b/scripts/conformance/rubyspec/cmd/specdelta/main.go index 18015d7b..88422669 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main.go @@ -842,11 +842,39 @@ func run(engine, root, file string, timeout time.Duration) (result, error) { // It also broke the control for the report as a whole. "grep KILLED; if 0 the // table is sound" is the documented way to tell whether the machine's load // decided a cell -- and that grep could only ever answer 0. - parsed, err := parse(out.String()) + parsed, err := parse(printable(out.String())) parsed.killed = r.killed return parsed, err } +// printable strips the control bytes a spec's own output can carry into this +// report. ruby/spec files print raw binary on purpose -- zlib round-trips, +// net-http bodies, String#unpack -- and the shim's RBGO_DETAIL line quotes the +// example's description and message verbatim. +// +// ⛔ MEASURED 2026-11-10. One library/ report came out holding NUL bytes, and +// `file` called it "RAGE Package Format (RPF)". Every grep over it then +// returned NOTHING AT ALL -- not zero matches, no output and exit 1, because +// grep skips a file it decides is binary. Three separate checks of that report +// ("is there a KILLED row?", "which files did not load?") were read as clean +// when the search had never looked inside. A report that ordinary tools refuse +// to read is a report whose silence cannot be told from a clean one. +// +// run.sh has stripped NULs since it was written, for the same reason and one +// layer further out (`tr -d '\000'`); this is that guard where the Go half can +// also rely on it. Tab and newline are kept, since the format is built on them. +func printable(s string) string { + return strings.Map(func(r rune) rune { + if r == '\t' || r == '\n' { + return r + } + if r < 0x20 || r == 0x7f { + return -1 + } + return r + }, s) +} + // parse reads the shim's report. It REFUSES a RESULT line it cannot read rather // than defaulting a field to zero, because a zero that means "unreadable" and a // zero that means "nothing passed" lead to opposite conclusions. diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go index 389671cf..92d379fb 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go @@ -913,3 +913,45 @@ func TestAKillIsReportedBeforeAnAbsence(t *testing.T) { }) } } + +// ⛔ A REPORT NO TOOL WILL READ. ruby/spec prints raw binary on purpose (zlib +// round-trips, net-http bodies, String#unpack), and the shim quotes an +// example's description verbatim into RBGO_DETAIL. Measured 2026-11-10: one +// library/ report came out holding NUL bytes, `file` called it "RAGE Package +// Format (RPF)", and every grep over it returned NOTHING -- no output, exit 1, +// because grep skips what it decides is binary. Three checks of that report +// were read as clean when the search had never looked inside it. +func TestControlBytesFromASpecDoNotReachTheReport(t *testing.T) { + const dirty = "RBGO_DETAIL\tfail\tZlib\tround\x00trips \x01binary\x7f\tmsg\n" + + "RBGO_RESULT pass=1 fail=1 error=0 skip=0\n" + r, err := parse(printable(dirty)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if !r.ran || r.pass != 1 { + t.Fatalf("the surrounding report must still parse: ran=%v pass=%d", r.ran, r.pass) + } + for label := range r.labels { + if strings.ContainsAny(label, "\x00\x01\x7f") { + t.Errorf("a control byte reached a label: %q", label) + } + } + // The text either side of the stripped byte must survive: dropping the whole + // description would lose the finding instead of the byte. + found := false + for label := range r.labels { + if strings.Contains(label, "roundtrips binary") { + found = true + } + } + if !found { + t.Errorf("the description did not survive the strip: %v", r.labels) + } +} + +// Tab and newline ARE the format and must not be stripped with the rest. +func TestPrintableKeepsTheFormatsOwnSeparators(t *testing.T) { + if got := printable("a\tb\nc"); got != "a\tb\nc" { + t.Errorf("printable mangled the separators: %q", got) + } +}