diff --git a/README.md b/README.md index b26535e1..416ed391 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ there is no honest way to turn it into a percentage of the language: measurement. (rbgo does bind a large set of stdlib and gem replacements — see *Supported today* — but their conformance is not what this figure reports.) - **The shim is not MSpec.** Exactly three helpers are stubbed — `ruby_exe` and - `ruby_cmd` (`spec_helper.rb:930-931`) and `output_to_fd` (`:306`) — so the + `ruby_cmd` (`spec_helper.rb:976-977`) and `output_to_fd` (`:341`) — so the examples that need them are counted as **skipped**, not passed. That makes the total a conservative lower bound. `complain`, `output` and `be_computed_by` are **implemented**; this list claimed otherwise until diff --git a/conformance_platform_guard_test.go b/conformance_platform_guard_test.go new file mode 100644 index 00000000..d93baae4 --- /dev/null +++ b/conformance_platform_guard_test.go @@ -0,0 +1,225 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package ruby_test + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The conformance shim must have ONE source for what platform this is (#837). +// +// It had two. `PLATFORMS = [:darwin, :bsd, :unix]` was a literal, and +// `PlatformGuard::PLATFORM = RUBY_PLATFORM` beside it was derived, under a +// comment asserting the literal was the single source of truth. On darwin the +// two agree, so the split is invisible from the machine the work is done on; +// on the ubuntu lane that enforces the gate, `platform_is :linux` was FALSE. +// +// ⛔ EVERY ASSERTION BELOW IS PLATFORM-INDEPENDENT, and it has to be: this +// test runs on ubuntu, macos and windows, and a test that spelled out an +// expected answer would be asserting which runner it is on. What it asserts +// instead is that the two sources AGREE, which is the actual invariant, plus +// three facts true of every machine there is. +// +// Each of them is red under the old shim on EVERY platform, which is what makes +// this a gate rather than a note. + +// platformProbe asks the shim and the guard class the same questions, then +// prints both answers. `platform_is` with no block returns its verdict +// (spec_helper.rb's `_guard_run` yields only when a block is given), so one run +// answers all of them. +const platformProbe = `require_relative '../spec_helper' +def say(label, v); puts "GUARD #{label}=#{!!v}"; end + +# The four names the old literal held, each against the class beside it. +say "is_linux", platform_is(:linux) +say "os_linux", PlatformGuard.os?(:linux) +say "is_darwin", platform_is(:darwin) +say "os_darwin", PlatformGuard.os?(:darwin) +say "is_bsd", platform_is(:bsd) +say "os_bsd", PlatformGuard.os?(:bsd) +say "is_unix", platform_is(:unix) +say "os_unix", PlatformGuard.os?(:unix) + +# The bits/bytes defect: the corpus writes 64, the shim compared it to +# WORDSIZE / 8. +say "ptr64", platform_is(pointer_size: 64) +say "os_ptr64", PlatformGuard.pointer_size?(64) +say "ptr32", platform_is(pointer_size: 32) +say "os_ptr32", PlatformGuard.pointer_size?(32) + +# The separate negation path: mspec is run_unless(match?) over ONE predicate. +say "not_linux_cl32", platform_is_not(:linux, c_long_size: 32) +say "match_linux_cl32", (PlatformGuard.os?(:linux) && PlatformGuard.c_long_size?(32)) + +# A full triple is a substring of itself; a list of symbols can never hold one. +say "triple", platform_is(RUBY_PLATFORM) +# ...and a name no machine carries is simply false. +say "unknown", platform_is(:definitely_not_a_platform) + +puts "PLATFORM=#{RUBY_PLATFORM}" + +describe "a platform probe" do + it "runs at all" do + 1.should == 1 + end +end +` + +func platformCorpus(t *testing.T) string { + t.Helper() + root := t.TempDir() + shim, err := os.ReadFile(shimRelPath) + if err != nil { + t.Fatalf("read the shim under test: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "spec_helper.rb"), shim, 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "core"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "core", "probe_spec.rb"), []byte(platformProbe), 0o644); err != nil { + t.Fatal(err) + } + return root +} + +// answers reads the probe's output back. A label the probe never printed is +// reported as missing rather than as false: "it did not answer" and "it +// answered no" need opposite responses, and this file's whole subject is a +// guard that silently answered no. +func answers(t *testing.T, out string) map[string]bool { + t.Helper() + got := map[string]bool{} + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + rest, ok := strings.CutPrefix(line, "GUARD ") + if !ok { + continue + } + label, value, ok := strings.Cut(rest, "=") + if !ok { + continue + } + got[label] = value == "true" + } + return got +} + +func mustHave(t *testing.T, got map[string]bool, label, out string) bool { + t.Helper() + v, ok := got[label] + if !ok { + t.Fatalf("the probe never answered %q, so this run proves nothing:\n%s", label, out) + } + return v +} + +// TestTheShimHasOneSourceForWhatPlatformThisIs is #837's gate. +func TestTheShimHasOneSourceForWhatPlatformThisIs(t *testing.T) { + if testing.Short() { + t.Skip("skipping the rbgo build in -short") + } + bin := buildRbgo(t) + corpus := platformCorpus(t) + + out := runProbe(t, bin, corpus, "") + + // The CONTROL, first: the probe really ran and really printed. Every + // assertion below reads a map, and an empty map agrees with anything. + if !strings.Contains(out, "PLATFORM=") { + t.Fatalf("the probe never reached its last line, so this run proves nothing:\n%s", out) + } + got := answers(t, out) + if len(got) == 0 { + t.Fatalf("the probe printed no GUARD lines at all:\n%s", out) + } + + // ⛔ THE INVARIANT. For each name, the guard helper and the guard class must + // answer alike. This does not say what the answer is, so it holds on every + // runner — and it is false on every runner under the old shim for :bsd and + // :unix, which no RUBY_PLATFORM this project builds for contains. + for _, name := range []string{"linux", "darwin", "bsd", "unix"} { + helper := mustHave(t, got, "is_"+name, out) + class := mustHave(t, got, "os_"+name, out) + if helper != class { + t.Errorf("platform_is(:%s) = %v but PlatformGuard.os?(:%s) = %v — two sources again (#837)\n%s", + name, helper, name, class, out) + } + } + + // The same invariant for the sizes, which is where the units diverged. + for _, size := range []string{"ptr64", "ptr32"} { + helper := mustHave(t, got, size, out) + class := mustHave(t, got, "os_"+size, out) + if helper != class { + t.Errorf("platform_is(pointer_size:) and PlatformGuard.pointer_size? disagree on %s: %v vs %v\n%s", + size, helper, class, out) + } + } + + // ⭐ And an absolute, because the two could in principle agree on being + // wrong: every machine has SOME pointer size. Under the old shim both of + // these were false, on every platform, so core/array/pack/j_spec.rb and + // core/string/unpack/j_spec.rb contributed nothing at all — measured at 0 + // in BASELINE, and 188 and 216 once the units matched. + if !mustHave(t, got, "ptr64", out) && !mustHave(t, got, "ptr32", out) { + t.Errorf("neither pointer_size: 64 nor pointer_size: 32 matches, so every such guard is shut:\n%s", out) + } + + // ⛔ ONE PREDICATE, NEGATED. mspec's platform_is_not is run_unless over the + // same match? platform_is runs on (guards/platform.rb:116-122). The old shim + // rebuilt the negation by OR-ing the parts, which is not its dual and + // differs exactly when a site carries a name AND an option — which this one + // does, on every platform. + notLinux := mustHave(t, got, "not_linux_cl32", out) + matchLinux := mustHave(t, got, "match_linux_cl32", out) + if notLinux != !matchLinux { + t.Errorf("platform_is_not(:linux, c_long_size: 32) = %v, want !%v — the negation is not the dual of the match\n%s", + notLinux, matchLinux, out) + } + + // A machine always matches its own triple, which mspec gets from substring + // matching and a list of symbols can never hold. True everywhere. + if !mustHave(t, got, "triple", out) { + t.Errorf("platform_is(RUBY_PLATFORM) is false, so the shim cannot match a full triple (core/process/clock_gettime_spec.rb:140 writes one)\n%s", out) + } + + // The accepting side of the same boundary: a guard that matched everything + // would satisfy every assertion above. + if mustHave(t, got, "unknown", out) { + t.Errorf("platform_is(:definitely_not_a_platform) is TRUE, so the guard matches anything\n%s", out) + } +} + +// TestTheShimDoesNotHoldAListOfPlatformNames keeps the fix from being undone by +// reintroducing the literal next to the derived source. It reads the file +// rather than running it, because a list that agrees with RUBY_PLATFORM on the +// machine the test runs on would satisfy every assertion above while putting +// the second source back for every other machine. +func TestTheShimDoesNotHoldAListOfPlatformNames(t *testing.T) { + b, err := os.ReadFile(shimRelPath) + if err != nil { + t.Fatalf("read %s: %v", shimRelPath, err) + } + src := string(b) + + // Read-the-right-file control: a wrong path would make the assertion below + // pass by vacuum. + if !strings.Contains(src, "class PlatformGuard") { + t.Fatalf("%s does not define PlatformGuard; this test is reading the wrong file (%d bytes)", shimRelPath, len(src)) + } + for _, line := range strings.Split(src, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + if strings.Contains(line, "PLATFORMS") { + t.Fatalf("%s names PLATFORMS in code: %q — the platform comes from RUBY_PLATFORM alone (#837)", shimRelPath, strings.TrimSpace(line)) + } + } +} diff --git a/scripts/conformance/rubyspec/README.md b/scripts/conformance/rubyspec/README.md index 19b876cf..35f318d6 100644 --- a/scripts/conformance/rubyspec/README.md +++ b/scripts/conformance/rubyspec/README.md @@ -319,6 +319,87 @@ Those 50 files are the surface any claim about the move has to cover. It remains the right instrument for the other half too: a `UPDATE_BASELINE=1` that absorbs a loss silently. +## Which examples the MACHINE selects — `cmd/platguard` + +`ruby_version_is` selects the population by version; `platform_is` selects it by +machine. Both decide whether examples run at all, so both are read the same way +and for the same reason. + +The shim had **two** sources for what platform this is, under a comment saying +it had one (#837): + +```ruby +PLATFORMS = [:darwin, :bsd, :unix] # a literal +class PlatformGuard + PLATFORM = RUBY_PLATFORM # derived +``` + +On darwin the two agree, so the split is **invisible from the machine the work +is done on**. On the ubuntu lane that enforces the gate, `platform_is :linux` +was false and `PlatformGuard.os?(:linux)` was true. + +mspec has no list either (`lib/mspec/guards/platform.rb:26-38`): every name but +`:windows` and `:wsl` is a plain substring of `RUBY_PLATFORM`. So the shim's +`platform_is` / `platform_is_not` now call `PlatformGuard`, there is no list, +and nothing is typed twice. `platform_is_not` is `!match?` over the **same** +predicate (`:116-122`), not a second negation rebuilt from the parts. + +Measured over the pinned corpus — 2206 files, 420 guard sites, 3 of which the +reader refuses rather than guesses at: + +| machine | skipped but should run | ran but should not | +|---|---|---| +| `x86_64-linux` | 20 guards, ≥29 examples, 19 files | 11 guards, ≥11 examples, 10 files | +| `arm64-darwin` | 2 guards, 1 file is whole-file | 1 guard, 1 example, 1 file | + +The two directions are never summed; one says the measurement is too small, the +other that it is of the wrong machine. + +### ⛔ The size options are in BITS, and comparing them to bytes shut both branches + +`PlatformGuard::POINTER_SIZE` is `RbConfig::SIZEOF["void*"] * 8` upstream +(`platform.rb:61-66`) — 64 on a 64-bit machine. The shim compared the corpus's +`pointer_size: 64` against `WORDSIZE / 8`, which is 8. So `pointer_size: 64` +**and** `pointer_size: 32` were both false, on every machine, and every guard +written that way was shut. Nothing looked wrong, because a guard being shut is +what a guard does. + +Two whole files are nothing but those guards. Measured with `cmd/shimdelta` on +darwin, same binary, same corpus, only the shim changed: + +```text + +188 core/array/pack/j_spec.rb (0 -> 188) + +216 core/string/unpack/j_spec.rb (0 -> 216) +``` + +Both were recorded as `0` in `BASELINE`, which is what a file contributing +nothing looks like from the outside. + +## Measuring a change to the SHIM — `cmd/shimdelta` + +`cmd/specdelta` compares two **engines** through one shim. That is the wrong +axis for a change to the shim's guards, where the engine is fixed and the shim +decides which examples exist. `shimdelta` runs one interpreter over the same +files under two `spec_helper.rb` and reports the move, which is what +[re-recording the baseline](#re-recording-the-baseline) requires be known first. + +```sh +go run ./cmd/shimdelta -rbgo -corpus \ + -control -treatment \ + -file core/array/pack/j_spec.rb +``` + +It refuses two byte-identical shims, counts a missing tally as `DIDNOTRUN` +rather than as zero, and prints the gained and lost totals on separate lines. + +⛔ It measures **passing** examples and nothing else, because the shim's file +channel carries the pass count alone (#843) and the four-field line lives on +stdout, which the judge must not read. A guard change that removes a *failure* +is therefore invisible to it — and that happens: on darwin `platform_is :bsd` +in `core/file/sticky_spec.rb` held one BSD-only expectation that was being +asserted on a mac, and fixing the guard took the file from `pass=4 fail=1` to +`pass=4 fail=0` while `shimdelta` correctly reported no change. + ## Making the reference interpreter able to answer — `cmd/mriref` ⛔ **An MRI that cannot load its bundled gems scores a clean ZERO on every file, @@ -672,9 +753,9 @@ what a ratchet should be: | stubbed (raises `SpecSkip`) | where | |---|---| -| `ruby_exe` — no subprocess | `spec_helper.rb:930` | -| `ruby_cmd` — no subprocess command line | `spec_helper.rb:931` | -| `output_to_fd` — no real fd redirection | `spec_helper.rb:306` | +| `ruby_exe` — no subprocess | `spec_helper.rb:976` | +| `ruby_cmd` — no subprocess command line | `spec_helper.rb:977` | +| `output_to_fd` — no real fd redirection | `spec_helper.rb:341` | ⛔ **That table is the whole list, and it is now derived rather than retyped.** It had a fourth row, "some `argf`/IO helpers", with no `where` — and nothing in @@ -687,7 +768,7 @@ top-level README name exactly that set. ⛔ **`complain`, `output` and `be_computed_by` are NOT stubbed**, and this paragraph said they were until 2026-11-08. They are implemented: -`ComplainMatcher` at `spec_helper.rb:230-262` with a real `$stderr` capture +`ComplainMatcher` at `spec_helper.rb:265-297` with a real `$stderr` capture (`complain` wired at `:249`), `OutputMatcher` at `:254-279`, and `be_computed_by` at `:199`. diff --git a/scripts/conformance/rubyspec/cmd/platguard/main.go b/scripts/conformance/rubyspec/cmd/platguard/main.go new file mode 100644 index 00000000..43b3a466 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/platguard/main.go @@ -0,0 +1,152 @@ +// platguard censuses ruby/spec's `platform_is` / `platform_is_not` guards and +// says which ones the conformance shim answered differently from mspec, on a +// named machine (#837). +// +// It is the static half of that measurement — the half that needs no sweep, and +// so can be run on a loaded machine, where a sweep cannot. The same reason +// cmd/guards exists for `ruby_version_is`. +// +// ⛔ WHY A NAMED MACHINE AND NOT THIS ONE. The gate runs on ubuntu and the +// developer runs on darwin, and the defect is INVISIBLE on darwin: the shim's +// hardcoded list happens to contain :darwin, so there the list and +// RUBY_PLATFORM agree. A tool that could only answer about its own host would +// have reported a clean corpus from the machine where the work is done. +// +// platguard -root [-platform x86_64-linux] [-list] +// +// Exit status: 0 the census ran and the two readings agree; 1 they disagree; +// 2 the corpus could not be read — a refusal, not an empty census. +package main + +import ( + "flag" + "fmt" + "os" + "sort" + "strings" + + gc "github.com/go-embedded-ruby/ruby/scripts/conformance/rubyspec/internal/guardcensus" +) + +// The shim's constants as they stood when #837 was filed, so the report says +// what the defect cost rather than what some other list would have cost. +var ( + oldPlatforms = []string{"darwin", "bsd", "unix"} + oldWordsize = 64 +) + +func main() { + root := flag.String("root", ".", "the ruby/spec checkout") + platform := flag.String("platform", "", "the RUBY_PLATFORM to answer about; empty means both machines the sweep runs on") + list := flag.Bool("list", false, "print every disagreeing site, not just the totals") + flag.Parse() + + // ⛔ language/ and core/ ONLY, named rather than filtered: the gate sweeps + // exactly two suites, so this walks exactly two. + sites, files, err := gc.WalkPlatform(*root, []string{"language", "core"}) + if err != nil { + fmt.Fprintf(os.Stderr, "platguard: walking %s: %v\n", *root, err) + os.Exit(2) + } + if files == 0 { + fmt.Fprintf(os.Stderr, "platguard: no *_spec.rb under %s/{language,core} -- not a ruby/spec checkout\n", *root) + os.Exit(2) + } + + unread := 0 + for _, s := range sites { + if s.Unclassified { + unread++ + } + } + + machines := []gc.Machine{gc.Linux64, gc.Darwin64} + if *platform != "" { + machines = []gc.Machine{{RubyPlatform: *platform, PointerSize: 64, CLongSize: 64}} + } + + out := os.Stdout + fmt.Fprintf(out, "corpus: %s — %d spec files, %d platform guard site(s)\n", *root, files, len(sites)) + if unread > 0 { + // Named, never folded into the totals: a site the reader could not + // parse is not a site the two readings agree about. + fmt.Fprintf(out, "⚠ %d site(s) could NOT be read and are judged by neither side\n", unread) + if *list { + for _, s := range sites { + if s.Unclassified { + fmt.Fprintf(out, " %s:%d %s\n", s.File, s.Line, s.Text) + } + } + } + } + + disagrees := false + for _, m := range machines { + ds := gc.Disagreements(sites, m, oldPlatforms, oldWordsize) + lostSites, gainedSites := 0, 0 + lostEx, gainedEx := 0, 0 + lostFiles, gainedFiles := map[string]bool{}, map[string]bool{} + for _, d := range ds { + if !d.Site.Block { + // A site used as a VALUE (`cmd = platform_is(:windows) ? …`) + // changes what an example asserts, not how many run. Counted + // apart for the same reason cmd/guards separates inline sites. + continue + } + if d.Lost() { + lostSites++ + lostEx += d.Site.Examples + lostFiles[d.Site.File] = true + } + if d.Gained() { + gainedSites++ + gainedEx += d.Site.Examples + gainedFiles[d.Site.File] = true + } + } + values := 0 + for _, d := range ds { + if !d.Site.Block { + values++ + } + } + if len(ds) > 0 { + disagrees = true + } + + fmt.Fprintf(out, "\n%s (pointer %d bits, c_long %d bits)\n", m.RubyPlatform, m.PointerSize, m.CLongSize) + fmt.Fprintf(out, " SKIPPED but should run : %4d guard(s) ≥%4d example(s) %3d file(s)\n", lostSites, lostEx, len(lostFiles)) + fmt.Fprintf(out, " RAN but should not : %4d guard(s) ≥%4d example(s) %3d file(s)\n", gainedSites, gainedEx, len(gainedFiles)) + fmt.Fprintf(out, " used as a VALUE : %4d site(s) — these change what an example asserts, not how many run\n", values) + // ⛔ The two directions are deliberately NOT summed. One says the + // measurement is too small; the other says it is of the wrong machine. + // A single number would let them cancel. + fmt.Fprintf(out, " ⛔ example counts are a LOWER BOUND: it_behaves_like expands at run time and a static reader cannot see it\n") + + if *list { + sorted := append([]gc.Disagreement{}, ds...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Site.File != sorted[j].Site.File { + return sorted[i].Site.File < sorted[j].Site.File + } + return sorted[i].Site.Line < sorted[j].Site.Line + }) + for _, d := range sorted { + verdict := "value" + if d.Site.Block { + if d.Lost() { + verdict = "SKIPPED→run" + } else { + verdict = "ran→SKIP" + } + } + fmt.Fprintf(out, " %-11s %s:%d (≥%d ex) %s\n", + verdict, d.Site.File, d.Site.Line, d.Site.Examples, strings.TrimSpace(d.Site.Text)) + } + } + } + + if disagrees { + os.Exit(1) + } +} diff --git a/scripts/conformance/rubyspec/cmd/shimdelta/main.go b/scripts/conformance/rubyspec/cmd/shimdelta/main.go new file mode 100644 index 00000000..307796df --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/shimdelta/main.go @@ -0,0 +1,331 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +// Command shimdelta measures what a change to the SHIM does to the population, +// by running ONE interpreter over the same files under two spec_helper.rb. +// +// Why it exists. cmd/specdelta compares two ENGINES through one shim, which is +// the right axis for "is rbgo behind MRI". It is the wrong axis for a change +// like #837, where the engine is fixed and the shim's guards decide which +// examples exist at all. Nothing here measured that, and the standing rule for +// such a change — written on #799 and repeated on #837 — is that the move must +// be measured BEFORE `UPDATE_BASELINE=1` regenerates anything, because the +// regeneration records whatever it sees and would absorb a loss silently. +// +// ⛔ A MISSING TALLY IS NOT A TALLY OF ZERO. The shim writes its pass count to +// RBGO_RESULT_FILE (#843) and writes nothing when it never reaches at_exit — a +// crash, a timeout, a file that would not load. Those come back as DIDNOTRUN +// and are counted in their own column, never as 0. A run where the control +// could not be measured is not evidence that the treatment gained anything. +// +// ⛔ IT MEASURES PASSING EXAMPLES AND NOTHING ELSE, and that is a real blind +// spot, stated rather than discovered later. The shim's file channel carries +// the pass count alone, deliberately (#843): the four-field line lives on +// stdout, which is the stream the subject can write to and therefore the one +// the judge must not read. So a guard change that removes a FAILURE is +// invisible here. It happens: on darwin, `platform_is :bsd` in +// core/file/sticky_spec.rb held exactly one example, a BSD-only expectation +// that was being asserted on a mac because :bsd sat in the shim's hardcoded +// list. Fixing the guard took that file from `pass=4 fail=1` to `pass=4 fail=0` +// and this command reported no change, correctly — the gate judges passing +// counts, and the passing count did not move. Read the engine's own output for +// a file whose fail column is the question. +// +// ⛔ IT REPORTS THE TWO DIRECTIONS SEPARATELY and refuses to print their sum. +// A guard fix both adds examples that should always have run and removes +// examples that were running on the wrong machine; a single net figure lets the +// two cancel, which is how a sweep once reported "+217 in rbgo's favour" twenty +// minutes before the same corpus read -1566. +// +// shimdelta -rbgo -corpus \ +// -control -treatment \ +// [-file ...] [-suites language,core] +// +// Paths given with -file are relative to the corpus. With no -file, every +// *_spec.rb under the named suites is run, which is a full sweep and takes as +// long as one. +// +// Exit status: 0 the two shims select the same population; 1 they differ; +// 2 the measurement could not be made — a refusal, not an empty delta. +package main + +import ( + "errors" + "flag" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +type fileList []string + +func (f *fileList) String() string { return strings.Join(*f, ",") } +func (f *fileList) Set(v string) error { + *f = append(*f, v) + return nil +} + +// result is one file under one shim. Ran distinguishes the two states the +// whole command exists to keep apart. +type result struct { + Ran bool + Pass int +} + +type row struct { + File string + Control, Treatment result +} + +func main() { + rbgo := flag.String("rbgo", "", "the interpreter binary to run under both shims (required)") + corpus := flag.String("corpus", "", "the ruby/spec checkout (required)") + control := flag.String("control", "", "the spec_helper.rb to measure FROM (required)") + treatment := flag.String("treatment", "", "the spec_helper.rb to measure TO (required)") + suites := flag.String("suites", "language,core", "comma-separated suites to walk when no -file is given") + timeout := flag.Duration("timeout", 60*time.Second, "per-file wall clock") + var files fileList + flag.Var(&files, "file", "a spec file relative to the corpus; repeatable") + flag.Parse() + + for name, v := range map[string]string{"-rbgo": *rbgo, "-corpus": *corpus, "-control": *control, "-treatment": *treatment} { + if v == "" { + fmt.Fprintf(os.Stderr, "shimdelta: %s is required\n", name) + os.Exit(2) + } + } + + controlSrc, err := os.ReadFile(*control) + if err != nil { + fmt.Fprintf(os.Stderr, "shimdelta: reading the control shim: %v\n", err) + os.Exit(2) + } + treatmentSrc, err := os.ReadFile(*treatment) + if err != nil { + fmt.Fprintf(os.Stderr, "shimdelta: reading the treatment shim: %v\n", err) + os.Exit(2) + } + // ⛔ A control and a treatment that are the same bytes cannot measure + // anything, and would print a tidy "no change" that looks like a verdict. + if string(controlSrc) == string(treatmentSrc) { + fmt.Fprintf(os.Stderr, "shimdelta: the two shims are byte-identical, so this run could not measure a difference\n") + os.Exit(2) + } + + targets := []string(files) + if len(targets) == 0 { + targets, err = walk(*corpus, strings.Split(*suites, ",")) + if err != nil { + fmt.Fprintf(os.Stderr, "shimdelta: walking the corpus: %v\n", err) + os.Exit(2) + } + } + if len(targets) == 0 { + fmt.Fprintf(os.Stderr, "shimdelta: no spec files to run under %s -- not a ruby/spec checkout\n", *corpus) + os.Exit(2) + } + + // ⛔ The shim is installed ONCE per arm and every file is run under it, + // rather than swapping the file between each pair. Swapping per file would + // make the two arms differ in when they ran as well as in which shim they + // used, and this machine's load is not constant. + shim := filepath.Join(*corpus, "spec_helper.rb") + saved, readErr := os.ReadFile(shim) + if readErr != nil { + saved = nil + } + defer func() { + if saved != nil { + _ = os.WriteFile(shim, saved, 0o644) + } + }() + + rows := make([]row, len(targets)) + for i, f := range targets { + rows[i].File = f + } + // ⛔ A SLICE, not a map: ranging a map randomises which arm runs first, and + // the arms would then differ in when they ran as well as in which shim they + // used. The control goes first, always, so the order is a stated constant + // rather than something the runtime picks. + for _, arm := range []struct { + name string + src []byte + }{{"control", controlSrc}, {"treatment", treatmentSrc}} { + if err := os.WriteFile(shim, arm.src, 0o644); err != nil { + fmt.Fprintf(os.Stderr, "shimdelta: installing the %s shim: %v\n", arm.name, err) + os.Exit(2) + } + for i, f := range targets { + r := runOne(*rbgo, *corpus, f, *timeout) + if arm.name == "control" { + rows[i].Control = r + } else { + rows[i].Treatment = r + } + } + } + + report(rows, *control, *treatment, len(targets)) +} + +// runOne runs one spec file and reads the tally from the shim's own channel. +// +// ⛔ The EXIT STATUS is not consulted and a failure to START is fatal to the +// measurement, because they are not the same thing: a spec with a failing +// example exits non-zero and has still been measured, while a binary that never +// ran has not. The tally file decides, and its absence is DIDNOTRUN. +func runOne(rbgo, corpus, file string, timeout time.Duration) result { + tally, err := os.CreateTemp("", "shimdelta-tally-") + if err != nil { + return result{} + } + path := tally.Name() + _ = tally.Close() + defer os.Remove(path) + + cmd := exec.Command(rbgo, file) + cmd.Dir = corpus + cmd.Env = append(os.Environ(), "RBGO_RESULT_FILE="+path) + cmd.Stdout, cmd.Stderr = nil, nil + + done := make(chan error, 1) + if err := cmd.Start(); err != nil { + fmt.Fprintf(os.Stderr, "shimdelta: could not run %s on %s: %v\n", rbgo, file, err) + os.Exit(2) + } + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + var exited *exec.ExitError + if err != nil && !errors.As(err, &exited) { + fmt.Fprintf(os.Stderr, "shimdelta: waiting on %s for %s: %v\n", rbgo, file, err) + os.Exit(2) + } + case <-time.After(timeout): + _ = cmd.Process.Kill() + <-done + return result{} + } + + b, err := os.ReadFile(path) + if err != nil || len(strings.TrimSpace(string(b))) == 0 { + return result{} + } + n, err := strconv.Atoi(strings.TrimSpace(string(b))) + if err != nil { + return result{} + } + return result{Ran: true, Pass: n} +} + +func walk(corpus string, suites []string) ([]string, error) { + var out []string + for _, suite := range suites { + suite = strings.TrimSpace(suite) + if suite == "" { + continue + } + dir := filepath.Join(corpus, suite) + if fi, err := os.Stat(dir); err != nil || !fi.IsDir() { + continue + } + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(p, "_spec.rb") { + return nil + } + rel, rerr := filepath.Rel(corpus, p) + if rerr != nil { + return rerr + } + out = append(out, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return nil, err + } + } + sort.Strings(out) + return out, nil +} + +// summary is what a run found, with the four outcomes kept apart. +type summary struct { + Gained, Lost []row // both arms measured the file and the count moved + Appeared []row // the control could not measure it, the treatment could + Unmeasured []row // at least one arm produced no tally at all + GainedN, LostN int +} + +// classify sorts the rows. +// +// ⛔ THE FIRST THREE CASES COME BEFORE ANY ARITHMETIC. A row whose tally is +// missing has a Pass of 0, so letting it fall through to the comparison would +// read "the control passed 0" — the exact substitution of an absent answer for +// a negative one that this whole command exists to refuse. A file neither arm +// could measure is Unmeasured, not an agreement at zero. +func classify(rows []row) summary { + var s summary + for _, r := range rows { + switch { + case r.Control.Ran && !r.Treatment.Ran: + s.Unmeasured = append(s.Unmeasured, r) + case !r.Control.Ran && r.Treatment.Ran: + s.Appeared = append(s.Appeared, r) + case !r.Control.Ran && !r.Treatment.Ran: + s.Unmeasured = append(s.Unmeasured, r) + case r.Treatment.Pass > r.Control.Pass: + s.Gained = append(s.Gained, r) + s.GainedN += r.Treatment.Pass - r.Control.Pass + case r.Treatment.Pass < r.Control.Pass: + s.Lost = append(s.Lost, r) + s.LostN += r.Control.Pass - r.Treatment.Pass + } + } + return s +} + +func report(rows []row, control, treatment string, files int) { + s := classify(rows) + gained, lost, appeared, vanished := s.Gained, s.Lost, s.Appeared, s.Unmeasured + gainedN, lostN := s.GainedN, s.LostN + + out := os.Stdout + fmt.Fprintf(out, "control : %s\n", control) + fmt.Fprintf(out, "treatment: %s\n", treatment) + fmt.Fprintf(out, "%d file(s) run under both\n\n", files) + + for _, r := range gained { + fmt.Fprintf(out, " +%-5d %s (%d -> %d)\n", r.Treatment.Pass-r.Control.Pass, r.File, r.Control.Pass, r.Treatment.Pass) + } + for _, r := range lost { + fmt.Fprintf(out, " -%-5d %s (%d -> %d)\n", r.Control.Pass-r.Treatment.Pass, r.File, r.Control.Pass, r.Treatment.Pass) + } + for _, r := range appeared { + fmt.Fprintf(out, " NEW %s (DID NOT RUN under the control -> %d)\n", r.File, r.Treatment.Pass) + } + for _, r := range vanished { + fmt.Fprintf(out, " ⛔ %s (control ran=%v, treatment ran=%v) -- NOT counted either way\n", r.File, r.Control.Ran, r.Treatment.Ran) + } + + // ⛔ The two totals are printed on their own lines and never added. A guard + // fix moves examples in both directions and the netting of the two is not a + // quantity anyone can act on. + fmt.Fprintf(out, "\n gained : %d example(s) across %d file(s)\n", gainedN, len(gained)) + fmt.Fprintf(out, " lost : %d example(s) across %d file(s)\n", lostN, len(lost)) + fmt.Fprintf(out, " unmeasured: %d file(s)\n", len(vanished)) + + if gainedN != 0 || lostN != 0 || len(appeared) != 0 || len(vanished) != 0 { + os.Exit(1) + } +} diff --git a/scripts/conformance/rubyspec/cmd/shimdelta/main_test.go b/scripts/conformance/rubyspec/cmd/shimdelta/main_test.go new file mode 100644 index 00000000..2ff9c227 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/shimdelta/main_test.go @@ -0,0 +1,110 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import "testing" + +// TestAMissingTallyIsNeverReadAsZero is the whole reason this command is not a +// shell loop around two sweeps. +// +// A file that crashed, timed out or would not load writes no tally, and the +// zero value of an unwritten count is 0 — the same byte a file that genuinely +// passed nothing produces. Letting those meet in the arithmetic turns "I could +// not measure the control" into "the control passed nothing, so the treatment +// gained everything", which is a plausible, coherent and false story about a +// run that measured nothing. +func TestAMissingTallyIsNeverReadAsZero(t *testing.T) { + for _, tc := range []struct { + name string + row row + wantBucket string + wantN int + }{ + { + "the control could not be measured, so nothing is attributed to the treatment", + row{File: "crashed.rb", Control: result{Ran: false}, Treatment: result{Ran: true, Pass: 188}}, + "appeared", 0, + }, + { + "a file that really passes nothing under both is NOT unmeasured", + row{File: "empty.rb", Control: result{Ran: true, Pass: 0}, Treatment: result{Ran: true, Pass: 0}}, + "none", 0, + }, + { + "the treatment stopped producing a tally — a loss of measurement, not of examples", + row{File: "broke.rb", Control: result{Ran: true, Pass: 12}, Treatment: result{Ran: false}}, + "unmeasured", 0, + }, + { + "neither arm measured it", + row{File: "dead.rb", Control: result{Ran: false}, Treatment: result{Ran: false}}, + "unmeasured", 0, + }, + { + "a real gain", + row{File: "j_spec.rb", Control: result{Ran: true, Pass: 0}, Treatment: result{Ran: true, Pass: 188}}, + "gained", 188, + }, + { + "a real loss", + row{File: "sticky.rb", Control: result{Ran: true, Pass: 4}, Treatment: result{Ran: true, Pass: 3}}, + "lost", 1, + }, + } { + t.Run(tc.name, func(t *testing.T) { + s := classify([]row{tc.row}) + got := map[string]int{ + "gained": len(s.Gained), + "lost": len(s.Lost), + "appeared": len(s.Appeared), + "unmeasured": len(s.Unmeasured), + } + for bucket, n := range got { + want := 0 + if bucket == tc.wantBucket { + want = 1 + } + if n != want { + t.Errorf("%s holds %d row(s), want %d", bucket, n, want) + } + } + if tc.wantBucket == "gained" && s.GainedN != tc.wantN { + t.Errorf("gained %d example(s), want %d", s.GainedN, tc.wantN) + } + if tc.wantBucket == "lost" && s.LostN != tc.wantN { + t.Errorf("lost %d example(s), want %d", s.LostN, tc.wantN) + } + // ⛔ The decisive one: an unmeasured row must contribute to NEITHER + // total. A row that landed in the right bucket and still moved a + // count would be the defect wearing the fix's label. + if tc.wantBucket == "unmeasured" || tc.wantBucket == "appeared" { + if s.GainedN != 0 || s.LostN != 0 { + t.Errorf("an unmeasured row moved the totals: gained=%d lost=%d", s.GainedN, s.LostN) + } + } + }) + } +} + +// TestTheTwoDirectionsAreCountedApart holds the other rule: a run that gains in +// one file and loses in another must report both, not their difference. A +// sweep of this corpus once read "+217 in rbgo's favour" twenty minutes before +// the same files read -1566. +func TestTheTwoDirectionsAreCountedApart(t *testing.T) { + s := classify([]row{ + {File: "up.rb", Control: result{Ran: true, Pass: 0}, Treatment: result{Ran: true, Pass: 188}}, + {File: "down.rb", Control: result{Ran: true, Pass: 190}, Treatment: result{Ran: true, Pass: 0}}, + }) + if s.GainedN != 188 { + t.Errorf("gained %d, want 188", s.GainedN) + } + if s.LostN != 190 { + t.Errorf("lost %d, want 190", s.LostN) + } + // Their difference is -2, which would read as a quiet, almost-clean run. + if len(s.Gained) != 1 || len(s.Lost) != 1 { + t.Fatalf("the two directions collapsed: %d gained, %d lost", len(s.Gained), len(s.Lost)) + } +} diff --git a/scripts/conformance/rubyspec/internal/guardcensus/platform.go b/scripts/conformance/rubyspec/internal/guardcensus/platform.go new file mode 100644 index 00000000..a63da3cc --- /dev/null +++ b/scripts/conformance/rubyspec/internal/guardcensus/platform.go @@ -0,0 +1,451 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package guardcensus + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// The platform half of the census (#837). +// +// `ruby_version_is` selects the population by VERSION; `platform_is` selects it +// by MACHINE. The two are read the same way and for the same reason: a guard +// decides whether examples run at all, so a shim that answers it differently +// from mspec is measuring a different corpus than the one it names. +// +// ⛔ WHAT THIS IS FOR. spec_helper.rb answered `platform_is` from a hardcoded +// list, `PLATFORMS = [:darwin, :bsd, :unix]`, while the class beside it derived +// `PLATFORM = RUBY_PLATFORM` — under a comment saying the list was the single +// source of truth. On darwin the two agree and the split is invisible; on the +// ubuntu lane that enforces the gate they do not. This reads the corpus and +// says, for a named machine, exactly which guards the two answers disagree +// about and how many examples sit behind them. +// +// The reference is ruby/mspec lib/mspec/guards/platform.rb, read at the tip on +// 2026-11-20: +// +// :19-24 PLATFORM = RUBY_PLATFORM (RbConfig's host triple under JRuby) +// :26-38 os?(*oses) — `:windows` is the regexp /(mswin|mingw)/, `:wsl` is +// its own predicate, and EVERY OTHER NAME is a plain substring of +// PLATFORM. There is no list of known systems anywhere in it. +// :61-66 POINTER_SIZE = RbConfig::SIZEOF["void*"] * 8 — in BITS +// :69-73 C_LONG_SIZE = RbConfig::SIZEOF["long"] * 8 — in BITS +// :97-113 match? — the platform names AND every option must all hold +// :116-122 platform_is is run_if(match?), platform_is_not run_unless(match?) +// +// ⛔ Note :116-122: the two entry points share ONE predicate, so +// `platform_is_not` is exactly `!match?`. A shim that re-derives the negation +// from the parts answers differently as soon as a site carries both a name and +// an option. + +// PlatformSite is one mention of platform_is or platform_is_not. +// +// A site whose argument list does not close on its own line is left +// Unclassified rather than guessed at: the corpus has at least one such call +// (core/kernel/system_spec.rb opens `platform_is(:windows, :darwin, :freebsd, +// :netbsd,` and continues on the next line), and a census that silently read +// the first line's names would report a smaller guard than the source holds. +// "Could not read it" and "read it and it names nothing" need opposite answers. +type PlatformSite struct { + File, Text string + Line int + + Negated bool // the call was platform_is_not + OSes []string // bare arguments, as written, without the leading colon + OptOSes []string // the `os:` option's arguments + PointerSize int // `pointer_size:` in BITS, 0 when absent + CLongSize int // `c_long_size:` in BITS, 0 when absent + Wordsize int // `wordsize:` in BITS, 0 when absent (deprecated upstream) + Block bool // it opens a block, so it SELECTS examples + Examples int // `it` blocks inside the block: a LOWER BOUND + Unclassified bool // the call could not be read; it is counted, never judged +} + +// Machine is the platform a census is asked about. The sizes are in BITS, +// because that is the unit mspec's POINTER_SIZE and C_LONG_SIZE carry +// (platform.rb:61-73) and mixing the two units is the defect this file was +// written to measure. +type Machine struct { + RubyPlatform string + PointerSize int + CLongSize int +} + +// Linux64 and Darwin64 are the two machines this project actually runs the +// sweep on: the gate's ubuntu lane, and the developer's mac. +// +// ⛔ The triples are what rbgo REPORTS, read from it rather than written from +// memory. internal/vm/kernel_introspect.go builds RUBY_PLATFORM from the Go +// build target, so it is `arm64-darwin` with no version suffix — MRI on the +// same machine says `arm64-darwin24`. Every name the corpus guards on is a +// substring of both, so the census is the same either way; the point is that +// the constant records a measurement and not a guess. +var ( + Linux64 = Machine{RubyPlatform: "x86_64-linux", PointerSize: 64, CLongSize: 64} + Darwin64 = Machine{RubyPlatform: "arm64-darwin", PointerSize: 64, CLongSize: 64} +) + +// OS reports whether a name matches this machine, as mspec's +// PlatformGuard.os? does (platform.rb:26-38). +func (m Machine) OS(name string) bool { + switch name { + case "windows": + return strings.Contains(m.RubyPlatform, "mswin") || strings.Contains(m.RubyPlatform, "mingw") + case "wsl": + // mspec shells out to `uname -r` for this one. The census answers for a + // named machine rather than for the host, so it cannot run that command, + // and a guess would be the fabricated positive this repository has + // already paid for twice. Treated as false and reported as such by + // cmd/platguard, which is what the sweep's own shim also answers. + return false + } + return strings.Contains(m.RubyPlatform, name) +} + +// Match is mspec's PlatformGuard#match? (platform.rb:97-113): the names, then +// every option, all of which must hold. +func (m Machine) Match(s PlatformSite) bool { + match := true + if len(s.OSes) > 0 { + match = false + for _, name := range s.OSes { + if m.OS(name) { + match = true + break + } + } + } + if len(s.OptOSes) > 0 { + any := false + for _, name := range s.OptOSes { + if m.OS(name) { + any = true + break + } + } + match = match && any + } + if s.PointerSize != 0 { + match = match && s.PointerSize == m.PointerSize + } + if s.CLongSize != 0 { + match = match && s.CLongSize == m.CLongSize + } + if s.Wordsize != 0 { + match = match && s.Wordsize == m.CLongSize + } + return match +} + +// Runs reports whether the examples behind this site execute on m, which is +// match? for platform_is and !match? for platform_is_not (platform.rb:116-122). +func (m Machine) Runs(s PlatformSite) bool { + if s.Negated { + return !m.Match(s) + } + return m.Match(s) +} + +// Literal is what spec_helper.rb answered BEFORE #837, kept so the cost of the +// defect can be measured rather than argued. It is a faithful transcription of +// the two guard bodies as they stood at spec_helper.rb:867-885: +// +// platform_is: a name matches when it is in the hardcoded PLATFORMS list; +// :pointer_size is compared against WORDSIZE/8 — BYTES against +// a value the corpus writes in BITS, so both 32 and 64 are +// false on every machine; +// the `os:` option is not read at all. +// platform_is_not: the negation is re-derived from the parts instead of being +// !match?, so a site carrying a name AND a size answers +// differently from mspec even when every part agrees. +// +// ⛔ It takes the list and the word size as arguments rather than reading +// constants, so a test can drive it with the values the shim actually had. +func Literal(s PlatformSite, platforms []string, wordsize int) bool { + inList := func(name string) bool { + for _, p := range platforms { + if p == name { + return true + } + } + return false + } + anyInList := func(names []string) bool { + for _, name := range names { + if inList(name) { + return true + } + } + return false + } + if s.Negated { + match := anyInList(s.OSes) + if s.Wordsize != 0 && s.Wordsize != wordsize { + match = true + } + if s.CLongSize != 0 && s.CLongSize != wordsize { + match = true + } + return !match + } + match := true + if len(s.OSes) > 0 { + match = anyInList(s.OSes) + } + if s.Wordsize != 0 { + match = match && s.Wordsize == wordsize + } + if s.PointerSize != 0 { + match = match && s.PointerSize == wordsize/8 + } + if s.CLongSize != 0 { + match = match && s.CLongSize == wordsize + } + return match +} + +var ( + platCallRe = regexp.MustCompile(`\bplatform_is(_not)?\b`) + platHeadRe = regexp.MustCompile(`\bplatform_is(_not)?\s*(\(|\s)`) + symRe = regexp.MustCompile(`^:([A-Za-z_][A-Za-z0-9_]*)$`) + strArgRe = regexp.MustCompile(`^["']([^"']*)["']$`) + optRe = regexp.MustCompile(`^([a-z_]+):\s*(.+)$`) +) + +// ClassifyPlatform reads one line of Ruby. It returns false when the line does +// not mention a platform guard at all. +func ClassifyPlatform(line string) (PlatformSite, bool) { + if !platCallRe.MatchString(line) { + return PlatformSite{}, false + } + s := PlatformSite{Text: strings.TrimSpace(line)} + loc := platHeadRe.FindStringSubmatchIndex(line) + if loc == nil { + s.Unclassified = true + return s, true + } + s.Negated = loc[2] != -1 + rest := line[loc[1]-1:] + args, body, ok := splitArgs(rest) + if !ok { + s.Unclassified = true + return s, true + } + s.Block = opensBlock(body) + for _, a := range args { + a = strings.TrimSpace(a) + if a == "" { + continue + } + if m := symRe.FindStringSubmatch(a); m != nil { + s.OSes = append(s.OSes, m[1]) + continue + } + if m := strArgRe.FindStringSubmatch(a); m != nil { + s.OSes = append(s.OSes, m[1]) + continue + } + m := optRe.FindStringSubmatch(a) + if m == nil { + s.Unclassified = true + return s, true + } + val := strings.TrimSpace(m[2]) + switch m[1] { + case "os": + if vm := symRe.FindStringSubmatch(val); vm != nil { + s.OptOSes = append(s.OptOSes, vm[1]) + } else { + s.Unclassified = true + return s, true + } + case "pointer_size", "c_long_size", "wordsize": + n, err := strconv.Atoi(val) + if err != nil { + s.Unclassified = true + return s, true + } + switch m[1] { + case "pointer_size": + s.PointerSize = n + case "c_long_size": + s.CLongSize = n + default: + s.Wordsize = n + } + default: + s.Unclassified = true + return s, true + } + } + return s, true +} + +// splitArgs takes the text from the call's name onwards and returns its +// arguments and whatever follows them. It reports false when the call does not +// finish on this line, which is a REFUSAL and not an empty argument list. +func splitArgs(rest string) (args []string, body string, ok bool) { + rest = strings.TrimSpace(rest) + if strings.HasPrefix(rest, "(") { + depth, end := 0, -1 + for i, r := range rest { + switch r { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + end = i + } + } + if end >= 0 { + break + } + } + if end < 0 { + return nil, "", false + } + return splitTop(rest[1:end]), rest[end+1:], true + } + // No parentheses: the arguments run to `do`, to `{`, to a `?`, to `and`/`or`, + // or to the end of the line. + cut := len(rest) + for _, kw := range []string{" do", " {", " ?", "?", " and ", " or ", " #"} { + if i := strings.Index(rest, kw); i >= 0 && i < cut { + cut = i + } + } + return splitTop(rest[:cut]), rest[cut:], true +} + +// splitTop splits an argument list on the commas that are not inside brackets. +func splitTop(s string) []string { + var out []string + depth, start := 0, 0 + for i, r := range s { + switch r { + case '(', '[', '{': + depth++ + case ')', ']', '}': + depth-- + case ',': + if depth == 0 { + out = append(out, s[start:i]) + start = i + 1 + } + } + } + out = append(out, s[start:]) + return out +} + +var platBlockRe = regexp.MustCompile(`^\s*(do\b|\{)`) + +func opensBlock(body string) bool { + return platBlockRe.MatchString(body) +} + +// CensusPlatform reads one spec file. +func CensusPlatform(path string, src []byte) []PlatformSite { + lines := strings.Split(strings.ReplaceAll(string(src), "\r\n", "\n"), "\n") + var out []PlatformSite + for i, line := range lines { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + s, ok := ClassifyPlatform(line) + if !ok { + continue + } + s.File, s.Line = path, i+1 + if s.Block { + s.Examples = ExamplesIn(lines, i) + } + out = append(out, s) + } + return out +} + +// WalkPlatform censuses every *_spec.rb under root/suite for each named suite, +// with File relative to root. The suites are NAMED, and the file count comes +// back, for the same reasons Walk gives. +func WalkPlatform(root string, suites []string) ([]PlatformSite, int, error) { + var sites []PlatformSite + files := 0 + for _, suite := range suites { + if fi, err := os.Stat(filepath.Join(root, suite)); err != nil || !fi.IsDir() { + continue + } + err := filepath.WalkDir(filepath.Join(root, suite), func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(p, "_spec.rb") { + return nil + } + files++ + b, rerr := os.ReadFile(p) + if rerr != nil { + return rerr + } + rel, rerr := filepath.Rel(root, p) + if rerr != nil { + return rerr + } + sites = append(sites, CensusPlatform(filepath.ToSlash(rel), b)...) + return nil + }) + if err != nil { + return nil, files, err + } + } + return sites, files, nil +} + +// Disagreement is one site the shim's old answer and mspec's answer differ +// about, on a named machine. +type Disagreement struct { + Site PlatformSite + Machine Machine + WasRun bool // the literal ran it + ShouldRun bool // mspec runs it +} + +// Lost reports whether examples that should run are being skipped. The other +// direction — examples running on a machine they were not written for — is +// Gained, and the two must never be summed: one is a measurement that is too +// small and the other is a measurement of the wrong thing. +func (d Disagreement) Lost() bool { return d.ShouldRun && !d.WasRun } + +// Gained reports the opposite direction. +func (d Disagreement) Gained() bool { return d.WasRun && !d.ShouldRun } + +// Disagreements compares the two readings over a census. +// +// ⛔ Unclassified sites are SKIPPED and counted separately by the caller. A +// site the reader could not parse is not a site the two readings agree about. +func Disagreements(sites []PlatformSite, m Machine, platforms []string, wordsize int) []Disagreement { + var out []Disagreement + for _, s := range sites { + if s.Unclassified { + continue + } + // Literal already answers for both entry points: it carries the shim's + // own, separate negation path rather than negating a shared predicate, + // which is one of the things being measured here. + was := Literal(s, platforms, wordsize) + should := m.Runs(s) + if was == should { + continue + } + out = append(out, Disagreement{Site: s, Machine: m, WasRun: was, ShouldRun: should}) + } + return out +} diff --git a/scripts/conformance/rubyspec/internal/guardcensus/platform_test.go b/scripts/conformance/rubyspec/internal/guardcensus/platform_test.go new file mode 100644 index 00000000..7107635c --- /dev/null +++ b/scripts/conformance/rubyspec/internal/guardcensus/platform_test.go @@ -0,0 +1,245 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package guardcensus + +import ( + "reflect" + "testing" +) + +// The shim's constants as they stood when #837 was filed. The tests drive +// Literal with these rather than with whatever the shim holds today, because +// Literal exists to say what the DEFECT cost, and a defect whose inputs move +// with the fix cannot be costed. +var ( + oldPlatforms = []string{"darwin", "bsd", "unix"} + oldWordsize = 64 +) + +func TestClassifyPlatformReadsTheShapesTheCorpusActuallyUses(t *testing.T) { + for _, tc := range []struct { + name string + line string + want PlatformSite + }{ + { + "the commonest site in the corpus", + " platform_is_not :windows do", + PlatformSite{Negated: true, OSes: []string{"windows"}, Block: true}, + }, + { + "several names", + " platform_is :linux, :darwin do", + PlatformSite{OSes: []string{"linux", "darwin"}, Block: true}, + }, + { + "a size option and no name", + " platform_is pointer_size: 64 do", + PlatformSite{PointerSize: 64, Block: true}, + }, + { + "c_long_size, which the shim already answered correctly", + " platform_is c_long_size: 32 do", + PlatformSite{CLongSize: 32, Block: true}, + }, + { + "parenthesised", + " platform_is_not(:windows) do", + PlatformSite{Negated: true, OSes: []string{"windows"}, Block: true}, + }, + { + // core/process/clock_gettime_spec.rb:140 writes a whole triple. + // mspec matches it as a substring like any other name, so a shim + // holding a list of symbols can never match it at all. + "a full platform triple as a string", + ` guard -> { platform_is "x86_64-linux" and kernel_version_is '3.0' } do`, + PlatformSite{OSes: []string{"x86_64-linux"}}, + }, + { + "used as a value, not as a block", + ` cmd = platform_is(:windows) ? 'rem' : 'cat'`, + PlatformSite{OSes: []string{"windows"}}, + }, + { + "a trailing comment does not swallow the block", + " platform_is :windows do #|| VMS", + PlatformSite{OSes: []string{"windows"}, Block: true}, + }, + { + // ⛔ core/file/stat/birthtime_spec.rb:3 opens its argument list and + // closes it on the NEXT line. Reading the first line's names would + // report a smaller guard than the source holds, so the site is + // refused instead. "Could not read it" is not "it names nothing". + "an argument list that does not close on its line is REFUSED", + "platform_is(:windows, :darwin, :freebsd, :netbsd,", + PlatformSite{Unclassified: true}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := ClassifyPlatform(tc.line) + if !ok { + t.Fatalf("ClassifyPlatform(%q) found no guard", tc.line) + } + got.Text, got.File, got.Line = "", "", 0 + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("ClassifyPlatform(%q)\n got %+v\nwant %+v", tc.line, got, tc.want) + } + }) + } +} + +// TestALineWithNoPlatformGuardIsNotASite is the accepting side of the reader's +// boundary: a gate that only ever finds something is not a measurement. +func TestALineWithNoPlatformGuardIsNotASite(t *testing.T) { + for _, line := range []string{ + ` ruby_version_is "4.0" do`, + ` it "works" do`, + ` @platform = "linux"`, + } { + if _, ok := ClassifyPlatform(line); ok { + t.Fatalf("ClassifyPlatform(%q) claimed a guard where there is none", line) + } + } +} + +// TestACommentedOutGuardIsNotCounted puts the comment rule where it actually +// lives. ClassifyPlatform answers about a line of code; CensusPlatform is what +// decides a line is code. The corpus contains at least one commented mention +// (core/io/close_on_exec_spec.rb carries a TODO naming `platform_is_not`), and +// counting it would inflate the census with a guard that does not exist. +func TestACommentedOutGuardIsNotCounted(t *testing.T) { + src := []byte("# TODO: remove this `platform_is_not` guard.\nplatform_is :linux do\n it \"x\" do\n end\nend\n") + sites := CensusPlatform("probe_spec.rb", src) + if len(sites) != 1 { + t.Fatalf("got %d site(s), want 1 — the commented line must not be counted: %+v", len(sites), sites) + } + if sites[0].Line != 2 { + t.Fatalf("the site is at line %d, want 2", sites[0].Line) + } + if sites[0].Examples != 1 { + t.Fatalf("the guard holds %d example(s), want 1", sites[0].Examples) + } +} + +// TestTheLiteralAndMspecDisagreeExactlyWhereTheySaidTheyWould is the heart of +// #837: each row is a site, and it carries BOTH answers. The rows where they +// agree are as load-bearing as the rows where they differ — a fix justified +// only by the failing rows could be a rule that is simply broader. +func TestTheLiteralAndMspecDisagreeExactlyWhereTheySaidTheyWould(t *testing.T) { + for _, tc := range []struct { + name string + line string + machine Machine + literalRuns bool + mspecRuns bool + }{ + { + "platform_is :linux on the gate's own runner — the headline of #837", + "platform_is :linux do", + Linux64, false, true, + }, + { + "and the same site on darwin, where both say no", + "platform_is :linux do", + Darwin64, false, false, + }, + { + "platform_is :darwin runs on LINUX because the literal contains it", + "platform_is :darwin do", + Linux64, true, false, + }, + { + "platform_is :darwin on darwin — both say yes", + "platform_is :darwin do", + Darwin64, true, true, + }, + { + // The literal's :bsd matches no RUBY_PLATFORM this project runs on. + // Measured: it was asserting a BSD-only expectation on a mac, and + // core/file/sticky_spec.rb went pass=4 fail=1 -> pass=4 fail=0. + "platform_is :bsd ran on darwin and should not", + "platform_is :bsd do", + Darwin64, true, false, + }, + { + // ⛔ The bits/bytes defect. The shim compared the corpus's 64 + // against WORDSIZE/8 = 8, so BOTH branches of every pointer_size + // guard were false and neither ran, on every machine. + "pointer_size: 64 never ran anywhere", + "platform_is pointer_size: 64 do", + Darwin64, false, true, + }, + { + "...nor on linux", + "platform_is pointer_size: 64 do", + Linux64, false, true, + }, + { + "and the 32-bit branch correctly stays shut, which is why nothing looked wrong", + "platform_is pointer_size: 32 do", + Linux64, false, false, + }, + { + // The separate negation path. mspec is run_unless(match?) over ONE + // predicate; the old shim OR-ed the parts, which is not its dual. + "platform_is_not with a name AND a size", + "platform_is_not :linux, c_long_size: 32 do", + Linux64, false, true, + }, + { + "c_long_size alone, which the shim already answered correctly", + "platform_is c_long_size: 64 do", + Linux64, true, true, + }, + { + "the corpus's commonest site, right by accident on both", + "platform_is_not :windows do", + Linux64, true, true, + }, + { + "a name neither side has heard of is simply false", + "platform_is :haiku do", + Linux64, false, false, + }, + { + "a full triple matches as a substring upstream and can never be in a list", + `platform_is "x86_64-linux" do`, + Linux64, false, true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + s, ok := ClassifyPlatform(tc.line) + if !ok || s.Unclassified { + t.Fatalf("ClassifyPlatform(%q) could not read the site", tc.line) + } + if got := Literal(s, oldPlatforms, oldWordsize); got != tc.literalRuns { + t.Errorf("the OLD shim ran it = %v, want %v", got, tc.literalRuns) + } + if got := tc.machine.Runs(s); got != tc.mspecRuns { + t.Errorf("mspec on %s runs it = %v, want %v", tc.machine.RubyPlatform, got, tc.mspecRuns) + } + }) + } +} + +// TestDisagreementsSkipsWhatItCouldNotRead holds the three states apart: a site +// the reader refused is neither an agreement nor a disagreement. +func TestDisagreementsSkipsWhatItCouldNotRead(t *testing.T) { + sites := []PlatformSite{ + {Unclassified: true, File: "unreadable.rb"}, + {OSes: []string{"linux"}, Block: true, File: "linux.rb"}, + {OSes: []string{"windows"}, Negated: true, Block: true, File: "agrees.rb"}, + } + ds := Disagreements(sites, Linux64, oldPlatforms, oldWordsize) + if len(ds) != 1 { + t.Fatalf("got %d disagreement(s), want exactly the one readable site that differs: %+v", len(ds), ds) + } + if ds[0].Site.File != "linux.rb" { + t.Fatalf("the disagreement is in %s, want linux.rb", ds[0].Site.File) + } + if !ds[0].Lost() || ds[0].Gained() { + t.Fatalf("platform_is :linux on linux was SKIPPED and should run: Lost=%v Gained=%v", ds[0].Lost(), ds[0].Gained()) + } +} diff --git a/scripts/conformance/rubyspec/spec_helper.rb b/scripts/conformance/rubyspec/spec_helper.rb index 66e13833..ac3ffecc 100644 --- a/scripts/conformance/rubyspec/spec_helper.rb +++ b/scripts/conformance/rubyspec/spec_helper.rb @@ -70,7 +70,23 @@ class SpecSkip < Exception; end # interpreter, the shim then measures at THAT interpreter's version, which is the # only answer that is ever right. CUR_VERSION = RUBY_VERSION -PLATFORMS = [:darwin, :bsd, :unix] + +# ⛔ THERE IS NO LIST OF PLATFORM NAMES, and that is the fix for #837. +# +# This line used to read `PLATFORMS = [:darwin, :bsd, :unix]`, a literal, while +# `PlatformGuard::PLATFORM = RUBY_PLATFORM` sat twenty lines below it under a +# comment saying the literal was the single source of truth. They CAN disagree, +# and on the ubuntu lane that enforces the gate they do: `PlatformGuard.os? +# (:linux)` is true there and `PLATFORMS.include?(:linux)` is false. On darwin +# the two agree, so the split is invisible from the machine the work is done on. +# +# mspec has no such list either (lib/mspec/guards/platform.rb:26-38): every name +# but `:windows` and `:wsl` is a plain substring of RUBY_PLATFORM. So the guards +# below ask PlatformGuard, and nothing here is typed twice. +# +# Measured with cmd/platguard over the pinned corpus (2206 files, 420 guard +# sites): on x86_64-linux the literal skipped 20 guards that should run and ran +# 11 that should not; on arm64-darwin24, 2 and 1. WORDSIZE = 64 ENDIAN = :little @@ -90,9 +106,17 @@ class SpecSkip < Exception; end end # mspec/lib/mspec/guards/platform.rb. Only the entry points the corpus reaches -# are provided; PLATFORMS/WORDSIZE above stay the single source of truth so the -# guard helpers and this class cannot disagree. +# are provided. This class is the ONLY place that decides what platform this is: +# the `platform_is` / `platform_is_not` helpers below call it rather than +# consulting a second source, so the two cannot disagree (#837). +# +# ⛔ THE SIZES ARE IN BITS, as mspec's are (platform.rb:61-73, both of them +# `RbConfig::SIZEOF[…] * 8`). The corpus writes `platform_is pointer_size: 64`, +# never 8. The helper below used to compare that 64 against `WORDSIZE / 8`, so +# BOTH the 64 and the 32 branch of every such guard were false and neither ran. class PlatformGuard + # mspec reads RbConfig's host triple under JRuby and RUBY_PLATFORM otherwise; + # only the second arm can be reached here. PLATFORM = RUBY_PLATFORM C_LONG_SIZE = WORDSIZE POINTER_SIZE = WORDSIZE @@ -101,7 +125,18 @@ def self.implementation?(*args) args.any? { |name| RUBY_ENGINE.start_with?(name == :rubinius ? 'rbx' : name.to_s) } end def self.standard?; implementation?(:ruby); end - def self.os?(*oses); oses.any? { |os| os == :windows ? !!(PLATFORM =~ /(mswin|mingw)/) : PLATFORM.include?(os.to_s) }; end + # platform.rb:26-38: :windows is a regexp over the triple, :wsl is its own + # predicate, and every other name is a plain substring. A name this build has + # never heard of is therefore simply false, not an error. + def self.os?(*oses) + oses.any? do |os| + case os + when :windows then !!(PLATFORM =~ /(mswin|mingw)/) + when :wsl then wsl? + else PLATFORM.include?(os.to_s) + end + end + end def self.windows?; os?(:windows); end def self.wasi?; os?(:wasi); end def self.wsl?; false; end @@ -864,25 +899,36 @@ def ruby_version_is(range, &b); _guard_run(_version_in_range(range), &b); end def version_is(base, range, &b); _guard_run(_version_in_range(range, base.to_s), &b); end def kernel_version_is(range, &b); _guard_run(_version_in_range(range, PlatformGuard.kernel_version), &b); end def ruby_bug(*a, &b); _guard_run(true, &b); end # assume bug fixed -def platform_is(*args, &b) - opts = args.last.is_a?(Hash) ? args.pop : {} - match = args.empty? ? true : args.any? { |s| PLATFORMS.include?(s) } - match &&= (opts[:wordsize].nil? || opts[:wordsize] == WORDSIZE) - match &&= (opts[:pointer_size].nil? || opts[:pointer_size] == WORDSIZE / 8) - # c_long_size guards examples written for a platform whose C long is narrower - # than this one. Ignoring it ran a 32-bit-only example here: - # "abc" * ((2 ** 31) - 1) is a legal six-gigabyte string on a 64-bit platform, - # and building it took 6.5 GB out of a CI runner's sixteen. - match &&= (opts[:c_long_size].nil? || opts[:c_long_size] == WORDSIZE) - _guard_run(match, &b) -end -def platform_is_not(*args, &b) +# mspec/lib/mspec/guards/platform.rb:97-122, transcribed. `match?` ANDs the +# platform names with every option, and the two entry points are `run_if` and +# `run_unless` over that ONE predicate. +# +# ⛔ ONE PREDICATE, NEGATED — never a second negation derived from the parts. +# The earlier `platform_is_not` rebuilt its own answer by OR-ing the parts +# together, which is not De Morgan's dual of this and diverges the moment a +# site carries a name AND an option: `platform_is_not :linux, c_long_size: 32` +# on 64-bit linux is `!(true && false)` = RUN here and upstream, and was +# SKIPPED by the old code. Sharing the predicate makes that unrepresentable. +# +# c_long_size guards examples written for a platform whose C long is narrower +# than this one. Ignoring it ran a 32-bit-only example here: +# "abc" * ((2 ** 31) - 1) is a legal six-gigabyte string on a 64-bit platform, +# and building it took 6.5 GB out of a CI runner's sixteen. +def _platform_match?(args) opts = args.last.is_a?(Hash) ? args.pop : {} - match = args.any? { |s| PLATFORMS.include?(s) } - match ||= (!opts[:wordsize].nil? && opts[:wordsize] != WORDSIZE) - match ||= (!opts[:c_long_size].nil? && opts[:c_long_size] != WORDSIZE) - _guard_run(!match, &b) + match = args.empty? ? true : PlatformGuard.os?(*args) + opts.each do |key, value| + case key + when :os then match &&= PlatformGuard.os?(*value) + when :pointer_size then match &&= PlatformGuard.pointer_size?(value) + when :wordsize then match &&= PlatformGuard.wordsize?(value) + when :c_long_size then match &&= PlatformGuard.c_long_size?(value) + end + end + match end +def platform_is(*args, &b); _guard_run(_platform_match?(args), &b); end +def platform_is_not(*args, &b); _guard_run(!_platform_match?(args), &b); end def not_supported_on(*engines, &b); _guard_run(!engines.include?(:ruby), &b); end def not_compliant_on(*engines, &b); _guard_run(!engines.include?(:ruby), &b); end def compliant_on(*engines, &b); _guard_run(engines.include?(:ruby), &b); end