Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 77 additions & 12 deletions scripts/conformance/rubyspec/cmd/specdelta/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,40 @@ type result struct {
// with one `skip` detail, entirely well-formed. Reading the shim rather than
// assuming its shape is what produced the table above; the line
// `hookerror` has no counter is a defect in the harness, filed separately.
// outcomeNote is the note printed beside a row when one of the two engines did
// not produce an ordinary result: it was stopped by the clock, or it produced
// nothing at all.
//
// ⛔ THE KILL IS TESTED FIRST, AND THE ORDER IS THE WHOLE POINT. A killed engine
// almost never prints a RESULT line, so `ran` is false and the DID-NOT-RUN arms
// would claim every killed row -- reporting a file the CLOCK stopped as a file
// the engine could not LOAD. Those are opposite findings: one says the timeout
// was too short or the engine hangs, the other says the feature is
// unimplemented, and they call for opposite work.
//
// It was worse than an ordering bug before this: run() returned `parse(...)`,
// a fresh result, so `killed` was false on every row ever printed and this
// branch was unreachable. The documented control for a whole report -- "grep
// for KILLED; if it is 0 the table is sound" -- could only ever answer 0, and
// every sweep looked clean by construction.
func outcomeNote(a, b result) string {
switch {
case a.killed && b.killed:
return " KILLED on timeout, both engines"
case a.killed:
return " KILLED on timeout, the control"
case b.killed:
return " KILLED on timeout, the treatment"
case !a.ran && !b.ran:
return " DID NOT RUN on either"
case !a.ran:
return " DID NOT RUN on the control"
case !b.ran:
return " DID NOT RUN on the treatment"
}
return ""
}

func (r result) tallyDisagrees() bool {
return r.ran && r.details < r.fail+r.errs
}
Expand Down Expand Up @@ -259,17 +293,7 @@ func main1(args []string, out, errOut io.Writer) int {
}
totalA += r.a.pass
totalB += r.b.pass
note := ""
switch {
case !r.a.ran && !r.b.ran:
note = " DID NOT RUN on either"
case !r.a.ran:
note = " DID NOT RUN on the control"
case !r.b.ran:
note = " DID NOT RUN on the treatment"
case r.a.killed || r.b.killed:
note = " KILLED on timeout"
}
note := outcomeNote(r.a, r.b)
if r.unstable {
note += " ⛔ UNSTABLE: an engine disagreed with itself, delta excluded"
}
Expand Down Expand Up @@ -807,7 +831,48 @@ func run(engine, root, file string, timeout time.Duration) (result, error) {
<-done
r.killed = true
}
return parse(out.String())
// ⛔ CARRY THE KILL ACROSS THE PARSE. This used to `return parse(...)`, which
// builds a FRESH result and threw `killed` away on the floor -- so `killed`
// was false on every row ever printed and the "KILLED on timeout" note was
// dead code. A killed engine usually prints no RESULT line either, so it fell
// through to `ran == false` and the report said "DID NOT RUN on the
// treatment": a HANG was published as an ABSENCE, which is a statement about
// conformance rather than about the clock, and the two call for opposite work.
//
// It also broke the control for the report as a whole. "grep KILLED; if 0 the
// table is sound" is the documented way to tell whether the machine's load
// decided a cell -- and that grep could only ever answer 0.
parsed, err := parse(printable(out.String()))
parsed.killed = r.killed
return parsed, err
}

// printable strips the control bytes a spec's own output can carry into this
// report. ruby/spec files print raw binary on purpose -- zlib round-trips,
// net-http bodies, String#unpack -- and the shim's RBGO_DETAIL line quotes the
// example's description and message verbatim.
//
// ⛔ MEASURED 2026-11-10. One library/ report came out holding NUL bytes, and
// `file` called it "RAGE Package Format (RPF)". Every grep over it then
// returned NOTHING AT ALL -- not zero matches, no output and exit 1, because
// grep skips a file it decides is binary. Three separate checks of that report
// ("is there a KILLED row?", "which files did not load?") were read as clean
// when the search had never looked inside. A report that ordinary tools refuse
// to read is a report whose silence cannot be told from a clean one.
//
// run.sh has stripped NULs since it was written, for the same reason and one
// layer further out (`tr -d '\000'`); this is that guard where the Go half can
// also rely on it. Tab and newline are kept, since the format is built on them.
func printable(s string) string {
return strings.Map(func(r rune) rune {
if r == '\t' || r == '\n' {
return r
}
if r < 0x20 || r == 0x7f {
return -1
}
return r
}, s)
}

// parse reads the shim's report. It REFUSES a RESULT line it cannot read rather
Expand Down
111 changes: 111 additions & 0 deletions scripts/conformance/rubyspec/cmd/specdelta/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"runtime"
"strings"
"testing"
"time"
)

// parse is where a silent wrong answer would live, so it is tested first and
Expand Down Expand Up @@ -844,3 +845,113 @@ func fakeEngine(t *testing.T, dir, name, body string) string {
}
return p
}

// ⛔ THE CONTROL THAT COULD NOT FIRE. run() used to `return parse(...)`, which
// builds a fresh result, so the `killed` it had just set was discarded. Every
// row printed carried killed == false, the "KILLED on timeout" note was
// unreachable, and the documented way to check a whole report -- grep it for
// KILLED, and trust the table when the count is 0 -- answered 0 by
// construction. Measured 2026-11-09: three sweeps of the ruby/spec corpus were
// declared sound on that grep.
func TestATimeoutIsCarriedOutOfRun(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("uses /bin/sleep as a deliberately slow engine")
}
// specdelta runs `engine <file>`, so /bin/sleep with "30" as the "spec file"
// is an engine that is guaranteed to outlive any timeout this test sets.
r, err := run("/bin/sleep", t.TempDir(), "30", 150*time.Millisecond)
if err != nil {
t.Fatalf("run: %v", err)
}
if !r.killed {
t.Error("an engine stopped by the timeout came back with killed == false; " +
"the report cannot then tell a hang from a file the engine could not load")
}
if r.ran {
t.Error("a killed engine printed no RESULT line, so ran must be false")
}
}

// An engine that finishes inside the timeout must NOT be marked killed, or the
// note would fire on every row and the control would be useless the other way.
func TestAnEngineThatFinishesIsNotMarkedKilled(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("uses /bin/sleep as a deliberately quick engine")
}
r, err := run("/bin/sleep", t.TempDir(), "0", 10*time.Second)
if err != nil {
t.Fatalf("run: %v", err)
}
if r.killed {
t.Error("an engine that exited on its own was reported as killed")
}
}

// ⛔ The ORDER of the note, which is what turned a hang into an absence. A
// killed engine prints no RESULT line, so ran is false too; if DID NOT RUN is
// tested first it claims every killed row.
func TestAKillIsReportedBeforeAnAbsence(t *testing.T) {
ranOK := result{ran: true, pass: 3}
killedSilent := result{killed: true} // what a real timeout looks like
for _, tc := range []struct {
name string
a, b result
want string
}{
{"the treatment was stopped by the clock", ranOK, killedSilent, " KILLED on timeout, the treatment"},
{"the control was stopped by the clock", killedSilent, ranOK, " KILLED on timeout, the control"},
{"both were", killedSilent, killedSilent, " KILLED on timeout, both engines"},
{"the treatment genuinely produced nothing", ranOK, result{}, " DID NOT RUN on the treatment"},
{"the control genuinely produced nothing", result{}, ranOK, " DID NOT RUN on the control"},
{"neither produced anything", result{}, result{}, " DID NOT RUN on either"},
{"both ran", ranOK, ranOK, ""},
} {
t.Run(tc.name, func(t *testing.T) {
if got := outcomeNote(tc.a, tc.b); got != tc.want {
t.Errorf("outcomeNote = %q, want %q", got, tc.want)
}
})
}
}

// ⛔ A REPORT NO TOOL WILL READ. ruby/spec prints raw binary on purpose (zlib
// round-trips, net-http bodies, String#unpack), and the shim quotes an
// example's description verbatim into RBGO_DETAIL. Measured 2026-11-10: one
// library/ report came out holding NUL bytes, `file` called it "RAGE Package
// Format (RPF)", and every grep over it returned NOTHING -- no output, exit 1,
// because grep skips what it decides is binary. Three checks of that report
// were read as clean when the search had never looked inside it.
func TestControlBytesFromASpecDoNotReachTheReport(t *testing.T) {
const dirty = "RBGO_DETAIL\tfail\tZlib\tround\x00trips \x01binary\x7f\tmsg\n" +
"RBGO_RESULT pass=1 fail=1 error=0 skip=0\n"
r, err := parse(printable(dirty))
if err != nil {
t.Fatalf("parse: %v", err)
}
if !r.ran || r.pass != 1 {
t.Fatalf("the surrounding report must still parse: ran=%v pass=%d", r.ran, r.pass)
}
for label := range r.labels {
if strings.ContainsAny(label, "\x00\x01\x7f") {
t.Errorf("a control byte reached a label: %q", label)
}
}
// The text either side of the stripped byte must survive: dropping the whole
// description would lose the finding instead of the byte.
found := false
for label := range r.labels {
if strings.Contains(label, "roundtrips binary") {
found = true
}
}
if !found {
t.Errorf("the description did not survive the strip: %v", r.labels)
}
}

// Tab and newline ARE the format and must not be stripped with the rest.
func TestPrintableKeepsTheFormatsOwnSeparators(t *testing.T) {
if got := printable("a\tb\nc"); got != "a\tb\nc" {
t.Errorf("printable mangled the separators: %q", got)
}
}
Loading