diff --git a/README.md b/README.md index 27d507f8..b26535e1 100644 --- a/README.md +++ b/README.md @@ -177,13 +177,13 @@ counters — agreed on every one of the 2,206 per-file records except there is no honest way to turn it into a percentage of the language: - **Only `language/` and `core/` are swept.** `run.sh` walks exactly those two - trees and no others (`scripts/conformance/rubyspec/run.sh:106`); ruby/spec's + trees and no others (`scripts/conformance/rubyspec/run.sh:118`); ruby/spec's `library/`, `optional/`, `command_line/` and `security/` trees are **not run at all**. Most of what people mean by "the standard library" is outside this measurement. (rbgo does bind a large set of stdlib and gem replacements — see *Supported today* — but their conformance is not what this figure reports.) - **The shim is not MSpec.** Exactly three helpers are stubbed — `ruby_exe` and - `ruby_cmd` (`spec_helper.rb:882-883`) and `output_to_fd` (`:280`) — so the + `ruby_cmd` (`spec_helper.rb:930-931`) and `output_to_fd` (`:306`) — so the examples that need them are counted as **skipped**, not passed. That makes the total a conservative lower bound. `complain`, `output` and `be_computed_by` are **implemented**; this list claimed otherwise until diff --git a/conformance_result_channel_test.go b/conformance_result_channel_test.go new file mode 100644 index 00000000..b713e5cb --- /dev/null +++ b/conformance_result_channel_test.go @@ -0,0 +1,290 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package ruby_test + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// The conformance ratchet's tally must come back through a channel the SUBJECT +// cannot write to (#843). +// +// It did not. The shim reported on stdout -- the same stream the spec writes to +// -- and run.sh fished the line back out with `grep '^RBGO_RESULT' | head -1`. +// The shim prints at exit, so anything the spec printed was FIRST: a file that +// failed every example while printing `RBGO_RESULT pass=9999 …` was recorded as +// `OK 9999`, and `UPDATE_BASELINE=1` banked that number. +// +// That needs no adversary. A spec for `puts`, for `$stdout` or for `String#%` +// can emit the prefix by accident; a SPEC_SHA bump pulls in whatever upstream +// wrote since; and run.sh's `2>&1` meant a WARNING text was enough. +// +// ⛔ These tests exist because reading the fix cannot distinguish it from the +// defect: both files mention RBGO_RESULT, both print a tally, and the whole +// difference is WHICH stream the judge believes. So the forgery is actually +// printed here, by a real spec, through the real binary. +const ( + shimRelPath = "scripts/conformance/rubyspec/spec_helper.rb" + runShRelPath = "scripts/conformance/rubyspec/run.sh" +) + +// forgingSpec prints the #843 probe's forged tally BEFORE its examples run, and +// also reports what the environment told it, so one run answers both halves: +// can the subject dictate the count, and can it even learn where the count goes. +// +// One example passes and one fails, so the true tally (pass=1) differs from the +// forgery (pass=9999) AND from zero -- a fix that wrote 0 for everything would +// also beat a forgery, and would be useless. +const forgingSpec = `require_relative '../spec_helper' +puts "RBGO_RESULT pass=9999 fail=0 error=0 skip=0" +puts "SAW_ENV=#{ENV["RBGO_RESULT_FILE"].inspect}" +describe "a probe" do + it "passes" do + 1.should == 1 + end + it "fails on purpose" do + 1.should == 2 + end +end +` + +// buildRbgo builds the real interpreter. A build failure FAILS, and never +// skips: a skip here would report green for a harness nobody could run (#687). +func buildRbgo(t *testing.T) string { + t.Helper() + bin := filepath.Join(t.TempDir(), "rbgo") + // `go build -o` writes EXACTLY the path given, so on Windows the result is + // an extensionless file that CreateProcess will not start. + if runtime.GOOS == "windows" { + bin += ".exe" + } + build := exec.Command("go", "build", "-o", bin, "./cmd/rbgo") + build.Env = append(os.Environ(), "GOWORK=off", "CGO_ENABLED=0") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("cannot build cmd/rbgo (%v): %s", err, out) + } + return bin +} + +// miniCorpus lays out a corpus the way ruby/spec is laid out -- the shim at the +// root, the spec one directory down reaching it with require_relative -- so the +// REAL shim file is what runs, not a copy of its ideas. +func miniCorpus(t *testing.T) string { + t.Helper() + root := t.TempDir() + shim, err := os.ReadFile(shimRelPath) + if err != nil { + t.Fatalf("read the shim under test: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "spec_helper.rb"), shim, 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "core"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "core", "probe_spec.rb"), []byte(forgingSpec), 0o644); err != nil { + t.Fatal(err) + } + return root +} + +// runProbe runs the forging spec. resultFile == "" means the variable is not set +// at all, which is the hand-run case. +// +// ⛔ A non-zero EXIT STATUS is ignored and a failure to START is fatal, and the +// two are not the same thing. A spec with a failing example is a normal outcome +// for this harness, so its status says nothing; but a binary that never ran +// returns empty output, and empty output makes every assertion below fail with +// the wrong reason. The first version of this helper wrote `out, _ :=` under a +// comment justifying only the first case, and the Windows lane then reported +// "the probe never printed its forged tally" for a build this test had named +// without `.exe`. "Could not ask" and "answered differently" need opposite +// responses -- the same rule cmd/oracle's own documentation states. +func runProbe(t *testing.T, bin, corpus, resultFile string) string { + t.Helper() + cmd := exec.Command(bin, "core/probe_spec.rb") + cmd.Dir = corpus + env := []string{} + for _, kv := range os.Environ() { + if !strings.HasPrefix(kv, "RBGO_RESULT_FILE=") { + env = append(env, kv) + } + } + if resultFile != "" { + env = append(env, "RBGO_RESULT_FILE="+resultFile) + } + cmd.Env = env + out, err := cmd.CombinedOutput() + var exited *exec.ExitError + if err != nil && !errors.As(err, &exited) { + t.Fatalf("could not run %s: %v\noutput so far:\n%s", bin, err, out) + } + return string(out) +} + +// TestASpecCannotDictateItsOwnRecordedPassCount is the #843 probe, run for real. +func TestASpecCannotDictateItsOwnRecordedPassCount(t *testing.T) { + if testing.Short() { + t.Skip("skipping the rbgo build in -short") + } + bin := buildRbgo(t) + corpus := miniCorpus(t) + result := filepath.Join(t.TempDir(), "tally") + + out := runProbe(t, bin, corpus, result) + + // The CONTROL, and the reason this test is not circular: the forgery was + // really printed. Were it merely absent -- a shim that stopped the spec + // from printing, a probe that never ran -- every assertion below would + // pass for the wrong reason. + if !strings.Contains(out, "RBGO_RESULT pass=9999") { + t.Fatalf("the probe never printed its forged tally, so this run proves nothing:\n%s", out) + } + + b, err := os.ReadFile(result) + if err != nil { + t.Fatalf("the shim wrote no tally file (%v); output was:\n%s", err, out) + } + if got := strings.TrimSpace(string(b)); got != "1" { + t.Fatalf("recorded pass count = %q, want \"1\" (the true count; 9999 is the forgery)\n%s", got, out) + } + + // The subject must not be able to LEARN the path either, or it could write + // the file instead of printing. The shim deletes the variable from ENV + // before any spec code runs. + if !strings.Contains(out, "SAW_ENV=nil") { + t.Fatalf("the spec could still read RBGO_RESULT_FILE from ENV:\n%s", out) + } + + // cmd/specdelta reads the four-field line off stdout and must keep working. + if !strings.Contains(out, "RBGO_RESULT pass=1 fail=1 error=0 skip=0") { + t.Fatalf("the shim's own tally line is missing or changed:\n%s", out) + } +} + +// TestAHandRunStillGetsItsTallyOnStdout is the other side of the boundary. The +// file channel is what the ratchet uses; a human running one spec file names no +// file and must still get the tally, so the stdout line is not a leftover to be +// deleted by someone tidying up after #843. +func TestAHandRunStillGetsItsTallyOnStdout(t *testing.T) { + if testing.Short() { + t.Skip("skipping the rbgo build in -short") + } + bin := buildRbgo(t) + corpus := miniCorpus(t) + unused := filepath.Join(t.TempDir(), "tally") + + out := runProbe(t, bin, corpus, "") + + if !strings.Contains(out, "RBGO_RESULT pass=1 fail=1 error=0 skip=0") { + t.Fatalf("a hand-run lost its tally:\n%s", out) + } + if _, err := os.Stat(unused); !os.IsNotExist(err) { + t.Fatalf("a file was written although none was named (stat err = %v)", err) + } +} + +// bannedExtractions are the three forms that used to pull the tally out of the +// subject's own stream. Each is a WHOLE command, not a word: `head` and `sed` +// have honest uses elsewhere in a shell script. +var bannedExtractions = []string{ + "grep '^RBGO_RESULT'", + "head -1", + "sed -E", +} + +// shellCodeHas answers whether a shell source RUNS the given text, ignoring +// full-line comments. +// +// ⛔ This exists because the first version of this gate did not have it, and the +// first thing it refused was the fix's own comment -- the line in run_one that +// names the `grep … | head -1` it replaced. The rule was wrong and the code was +// right. Banning a string anywhere in the file would forbid ever DOCUMENTING +// the defect, which is the opposite of what #843 asks for. +// +// A comment is a line whose first non-blank character is '#'. A trailing +// comment after code still counts as code, which errs toward refusing: a false +// alarm costs a reword, a miss costs the measurement. +func shellCodeHas(src, needle string) bool { + for _, line := range strings.Split(src, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + if strings.Contains(line, needle) { + return true + } + } + return false +} + +// TestShellCodeHasSeesCodeAndNotComments gives the comparison above a witness on +// EACH side of its boundary, because a gate that can only refuse, or only +// accept, is not a measurement. The accepting rows are the ones that catch a +// formulation that is too broad. +func TestShellCodeHasSeesCodeAndNotComments(t *testing.T) { + for _, tc := range []struct { + name string + src string + want bool + }{ + {"the extraction itself", "res=\"$(printf '%s\\n' \"$out\" | grep '^RBGO_RESULT' | head -1)\"", true}, + {"indented code still counts", " p=\"$(echo x | sed -E 's/a/b/')\"", true}, + {"a comment naming it does not", " # the grep '^RBGO_RESULT' | head -1 that used to live here is gone", false}, + {"a comment at column zero does not", "# sed -E 's/.*pass=([0-9]+).*/\\1/'", false}, + {"a trailing comment is treated as code", "cat \"$rf\" # not like head -1", true}, + {"an unrelated file is clean", "echo hello\ncat \"$rf\"\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := shellCodeHas(tc.src, firstMatching(tc.src)); got != tc.want { + t.Fatalf("shellCodeHas(%q) = %v, want %v", tc.src, got, tc.want) + } + }) + } +} + +// firstMatching picks whichever banned form the case is about, so the table +// above says what it means instead of repeating the needle in every row. A row +// mentioning none of them is asked about the first, which must then be absent. +func firstMatching(src string) string { + for _, b := range bannedExtractions { + if strings.Contains(src, b) { + return b + } + } + return bannedExtractions[0] +} + +// TestTheRatchetReadsTheTallyFromItsOwnFile gates the shell half. The judge is +// Go and is tested as Go; run.sh is what hands it the numbers, and a revert to +// `grep | head -1` there would restore the defect while every Go test above +// stayed green -- the shim's file channel would simply go unread. +func TestTheRatchetReadsTheTallyFromItsOwnFile(t *testing.T) { + b, err := os.ReadFile(runShRelPath) + if err != nil { + t.Fatalf("read %s: %v", runShRelPath, err) + } + src := string(b) + + // Read-the-right-file control: an empty or wrong path would make every + // "must not run" assertion below pass by vacuum. + if !shellCodeHas(src, "run_one()") { + t.Fatalf("%s does not define run_one; this test is reading the wrong file (%d bytes)", runShRelPath, len(src)) + } + + if !shellCodeHas(src, "RBGO_RESULT_FILE=") { + t.Fatalf("%s never sets RBGO_RESULT_FILE, so the tally is coming from somewhere the spec can write", runShRelPath) + } + for _, banned := range bannedExtractions { + if shellCodeHas(src, banned) { + t.Fatalf("%s still extracts the tally from the subject's stream: found %q (#843)", runShRelPath, banned) + } + } +} diff --git a/scripts/conformance/rubyspec/README.md b/scripts/conformance/rubyspec/README.md index 9bde0ea5..19b876cf 100644 --- a/scripts/conformance/rubyspec/README.md +++ b/scripts/conformance/rubyspec/README.md @@ -9,8 +9,32 @@ Ruby language and core library). `run.sh` runs the `language/` and `core/` suites of ruby/spec — pinned to the commit in `SPEC_SHA` — through rbgo, under a minimal MSpec-compatible shim (`spec_helper.rb`, installed in place of ruby/spec's real `spec_helper.rb`). -Each spec file runs in its own `rbgo` process; the shim prints a -`RBGO_RESULT pass=.. fail=.. error=.. skip=..` line at exit. +Each spec file runs in its own `rbgo` process. At exit the shim reports twice, +and the two reports are **not interchangeable**: + +| channel | content | who reads it | +|---|---|---| +| the file named by `RBGO_RESULT_FILE` | the pass count, decimal digits and nothing else | `run.sh`, for the ratchet | +| stdout | `RBGO_RESULT pass=.. fail=.. error=.. skip=..`, then the `RBGO_DETAIL` lines | a human, and `cmd/specdelta` | + +⛔ The file exists because the stdout line is **written into the stream the +subject also writes to**, and `run.sh` used to fish it back out with +`grep '^RBGO_RESULT' | head -1`. The shim prints at exit, so anything the spec +printed came first: a file that failed every example while printing +`RBGO_RESULT pass=9999 …` was recorded as `OK 9999`, and +`UPDATE_BASELINE=1` banked that number (measured 2026-10-31; issue #843). + +No adversary is needed for that. A spec for `puts`, for `$stdout` or for +`String#%` can emit the prefix by accident, a `SPEC_SHA` bump pulls in whatever +upstream has written since, and the old `2>&1` meant a *warning* text was +enough. + +`run.sh` now names a path per spec file and the shim **deletes +`RBGO_RESULT_FILE` from `ENV`** before any spec code runs, so the subject can +neither print the tally nor learn where it goes. Reading it is `cat`: the +`grep | head -1` and the `sed -E 's/.*pass=([0-9]+).*/\1/'` are gone, and with +them the last place where a not-quite-matching line produced a non-numeric +count. The sweep is then handed to `cmd/ratchet`, which judges it **file by file** against `BASELINE`: @@ -545,7 +569,14 @@ It keys on the **description** and not the message, because a message carries object addresses that differ between two runs of the *same* binary. **Two channels.** `RBGO_RESULT pass=…` comes out of the subject's own stdout, so -a subject that miscounts can claim anything. specdelta also counts the +a subject that miscounts can claim anything. (specdelta still reads that stream, +because the four-field tally and the `RBGO_DETAIL` lines it needs are only +there — `RBGO_RESULT_FILE` carries the pass count alone. Its guard against a +forged line is to refuse **two** result lines in one run, which is exactly what +a forgery produces alongside the shim's own. The ratchet, which needs only the +one number, moved off the stream entirely instead: see *What it does* and #843. +Giving specdelta the same out-of-band channel would mean widening the file's +format, and is not part of that change.) specdelta also counts the `RBGO_DETAIL` lines and flags a file that printed **fewer** than its own `fail + error` tally. It is an inequality and not an equality, because the shim emits a detail with no counter for a `before(:all)` failure and a counter with @@ -641,9 +672,9 @@ what a ratchet should be: | stubbed (raises `SpecSkip`) | where | |---|---| -| `ruby_exe` — no subprocess | `spec_helper.rb:904` | -| `ruby_cmd` — no subprocess command line | `spec_helper.rb:905` | -| `output_to_fd` — no real fd redirection | `spec_helper.rb:280` | +| `ruby_exe` — no subprocess | `spec_helper.rb:930` | +| `ruby_cmd` — no subprocess command line | `spec_helper.rb:931` | +| `output_to_fd` — no real fd redirection | `spec_helper.rb:306` | ⛔ **That table is the whole list, and it is now derived rather than retyped.** It had a fourth row, "some `argf`/IO helpers", with no `where` — and nothing in @@ -656,7 +687,7 @@ top-level README name exactly that set. ⛔ **`complain`, `output` and `be_computed_by` are NOT stubbed**, and this paragraph said they were until 2026-11-08. They are implemented: -`ComplainMatcher` at `spec_helper.rb:204-249` with a real `$stderr` capture +`ComplainMatcher` at `spec_helper.rb:230-262` with a real `$stderr` capture (`complain` wired at `:249`), `OutputMatcher` at `:254-279`, and `be_computed_by` at `:199`. diff --git a/scripts/conformance/rubyspec/run.sh b/scripts/conformance/rubyspec/run.sh index 5eb39dd4..d36b50ff 100755 --- a/scripts/conformance/rubyspec/run.sh +++ b/scripts/conformance/rubyspec/run.sh @@ -27,10 +27,12 @@ # file not loading, or 43 specs regressing, and those need opposite responses. # Judged per file, no margin is needed and every move names its file. # -# Each spec file runs in its own rbgo process (the shim prints a -# `RBGO_RESULT pass=.. fail=.. error=.. skip=..` line at exit); a file that -# crashes, times out or fails to load contributes 0 and is listed. The corpus is -# pinned to SPEC_SHA for reproducibility. +# Each spec file runs in its own rbgo process and reports its pass count +# through RBGO_RESULT_FILE -- a path this script names and the spec is never +# told, because a tally read out of the subject's own stdout is a tally the +# subject can dictate (#843). A file that crashes, times out or fails to load +# writes nothing, contributes 0 and is listed. The corpus is pinned to SPEC_SHA +# for reproducibility. # # Env: # RBGO path to the rbgo binary (default: built from ./cmd/rbgo) @@ -87,16 +89,26 @@ trap 'rm -f "$RESULTS"' EXIT run_one() { f="$1" - # Strip NULs: some specs print binary data, which bash command substitution - # would warn about ("ignored null byte in input"). - out="$(cd "$SPECDIR" && timeout "$TIMEOUT" "$RBGO" "$f" 2>&1 | tr -d '\000')" - res="$(printf '%s\n' "$out" | grep '^RBGO_RESULT' | head -1)" - if [ -z "$res" ]; then - printf 'FILEFAIL\t%s\n' "${f#"$SPECDIR"/}" + # ⛔ The tally comes back through a file, NOT through the subject's stdout + # (#843). The shim reads RBGO_RESULT_FILE and deletes it from ENV before any + # spec code runs, so the spec can neither learn this path nor write it; a + # spec that prints `RBGO_RESULT pass=9999 …` no longer dictates its own + # recorded count. The file holds the pass count and nothing else, so reading + # it is `cat` — the `grep '^RBGO_RESULT' | head -1` and the + # `sed -E 's/.*pass=([0-9]+).*/\1/'` that used to live here are gone, and + # with them the last place where a not-quite-matching line yielded a + # non-numeric count. + rf="$(mktemp)" + (cd "$SPECDIR" && RBGO_RESULT_FILE="$rf" timeout "$TIMEOUT" "$RBGO" "$f" >/dev/null 2>&1) + if [ -s "$rf" ]; then + printf 'OK\t%s\t%s\n' "${f#"$SPECDIR"/}" "$(cat "$rf")" else - p="$(printf '%s' "$res" | sed -E 's/.*pass=([0-9]+).*/\1/')" - printf 'OK\t%s\t%s\n' "${f#"$SPECDIR"/}" "$p" + # Absent or empty: a crash, a timeout, a file that would not load, or a + # shim that never reached at_exit. All of them are FILEFAIL, which is the + # behaviour this already had for a missing result line. + printf 'FILEFAIL\t%s\n' "${f#"$SPECDIR"/}" fi + rm -f "$rf" } export -f run_one export SPECDIR RBGO TIMEOUT diff --git a/scripts/conformance/rubyspec/spec_helper.rb b/scripts/conformance/rubyspec/spec_helper.rb index 2b3b840e..66e13833 100644 --- a/scripts/conformance/rubyspec/spec_helper.rb +++ b/scripts/conformance/rubyspec/spec_helper.rb @@ -19,6 +19,32 @@ class SpecSkip < Exception; end $mock_registry = nil $mock_installs = nil +# ⛔ THE TALLY'S OWN CHANNEL, which the subject cannot write to (#843). +# +# The `RBGO_RESULT` line below goes to stdout — the same stream the spec writes +# to — and the ratchet used to fish it back out with `grep … | head -1`. The +# shim prints at exit, so ANYTHING the spec printed is first: a spec containing +# `puts "RBGO_RESULT pass=9999 fail=0 error=0 skip=0"` was recorded as 9999 +# while failing every example, and `UPDATE_BASELINE=1` banked that number. +# Measured 2026-10-31 with a probe in the corpus; it is quoted on #843 rather +# than committed. +# +# That needs no adversary. A spec for `puts`, for `$stdout` or for `String#%` +# can emit the prefix by accident, a `SPEC_SHA` bump pulls in whatever upstream +# wrote since, and the harness's `2>&1` means a WARNING text is enough. +# +# So when the harness names a file, the tally goes THERE and the subject is +# never told where. The name is read once, here, before any spec code runs, and +# DELETED from the environment in the same breath: a spec that prints +# `ENV["RBGO_RESULT_FILE"]` gets nil, so it cannot write the file either. (That +# `ENV.delete` returns the value and leaves `nil` behind was measured against +# MRI 4.0.7 with cmd/oracle: both engines agreed byte for byte.) +# +# The stdout line STAYS, unchanged, for a hand-run and for cmd/specdelta, whose +# own reader already refuses two `RBGO_RESULT` lines in one run. It is now +# HUMAN output; the file is the protocol. +$RB_RESULT_FILE = ENV.delete("RBGO_RESULT_FILE") + # ⛔ DERIVED, never typed again (#799). This selects the POPULATION the whole # conformance figure is counted over: at 3.4.1 every example behind a # `ruby_version_is "4.0"` guard is removed from the suite before anything runs @@ -1181,6 +1207,17 @@ def evaluate(str, desc = nil, &block) at_exit do $stdout.puts "RBGO_RESULT pass=#{$RB_PASS} fail=#{$RB_FAIL} error=#{$RB_ERROR} skip=#{$RB_SKIP}" + # The ratchet's channel (#843). Deliberately the pass count ALONE, as decimal + # digits and nothing else, because the shell's only remaining job is to hand + # the bytes to the judge: no prefix to match, no field to extract, nothing + # that can half-match. The four-field tally stays on stdout above, where + # cmd/specdelta reads it. + # + # Written AFTER that line, so a failure here cannot cost specdelta its input. + # Unwritable is left to raise: every file would then come back as FILEFAIL, + # which is loud, and a rescue here would make a broken harness look like an + # engine that scores zero. + File.write($RB_RESULT_FILE, "#{$RB_PASS}\n") if $RB_RESULT_FILE $RB_FAILS.each do |kind, dsc, itn, cls, msg| $stdout.puts "RBGO_DETAIL\t#{kind}\t#{cls}\t#{(dsc.to_s + ' | ' + itn.to_s).gsub(/\s+/, ' ')[0, 160]}\t#{msg.to_s.gsub(/\s+/, ' ')[0, 160]}" end