Repository navigation
Expand file tree
/
Copy pathrun.sh
More file actions
executable file
·137 lines (126 loc) · 5.98 KB
/
Copy pathrun.sh
File metadata and controls
executable file
·137 lines (126 loc) · 5.98 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
#!/usr/bin/env bash
# Copyright (c) the go-embedded-ruby/ruby authors
#
# SPDX-License-Identifier: BSD-3-Clause
#
# ruby/spec conformance RATCHET for go-embedded-ruby (rbgo).
#
# Runs the ruby/spec language + core suites through rbgo under a minimal
# MSpec-compatible shim (spec_helper.rb, vendored beside this script) and
# judges the result against a PER-FILE baseline (the BASELINE file beside this
# script). It is a shrink-only ratchet: if any file passes fewer examples than
# its recorded count, or stops loading, the run FAILS -- so conformance is
# tracked and can only go up. When rbgo improves, regenerate the baseline with
# UPDATE_BASELINE=1 to lock the win in.
#
# The judging is a Go program (cmd/ratchet beside this script), not more shell.
# Reading a sweep and deciding is where a silent failure reads as a clean
# result, and the tool refuses input it cannot read -- an empty sweep, an
# unrecognised record, a pass count that is not a number -- with exit 2, which
# is distinct from exit 1 for a real regression.
#
# It replaced a single frozen scalar, which could not be calibrated. A spec file
# that fails to load carries tens of examples -- one flip was measured moving the
# total by 43 on a documentation-only commit -- so the floor needed a margin
# wider than the largest file, and a gate with a 43-example margin is blind to
# every regression smaller than 43. The total was also unattributable: -43 is one
# file not loading, or 43 specs regressing, and those need opposite responses.
# Judged per file, no margin is needed and every move names its file.
#
# Each spec file runs in its own rbgo process and reports its pass count
# through RBGO_RESULT_FILE -- a path this script names and the spec is never
# told, because a tally read out of the subject's own stdout is a tally the
# subject can dictate (#843). A file that crashes, times out or fails to load
# writes nothing, contributes 0 and is listed. The corpus is pinned to SPEC_SHA
# for reproducibility.
#
# Env:
# RBGO path to the rbgo binary (default: built from ./cmd/rbgo)
# SPECDIR ruby/spec checkout (default: cloned to CACHE at SPEC_SHA)
# CACHE clone cache dir (default: /tmp/rubyspec-corpus)
# JOBS parallelism (default: number of CPUs)
# TIMEOUT per-file timeout seconds (default: 25)
# UPDATE_BASELINE=1 rewrite BASELINE from this sweep and exit 0
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "$HERE/../../.." && pwd)"
RBGO="${RBGO:-/tmp/rbgo-rubyspec}"
CACHE="${CACHE:-/tmp/rubyspec-corpus}"
SPECDIR="${SPECDIR:-$CACHE}"
TIMEOUT="${TIMEOUT:-25}"
JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}"
# The ruby/spec commit the baseline was measured against. Bump together with
# BASELINE: a corpus bump changes which files exist, and the judge reports files
# that appear or vanish rather than silently absorbing them.
SPEC_SHA="87b1631992bd00cf0c4934474766d54dad088191"
SPEC_URL="https://github.com/ruby/spec"
BASELINE="$HERE/BASELINE"
echo "== build rbgo =="
if [ ! -x "$RBGO" ]; then
(cd "$REPO_ROOT" && GOWORK=off CGO_ENABLED=0 go build -o "$RBGO" ./cmd/rbgo) || exit 1
fi
echo "rbgo: $RBGO"
echo
echo "== ruby/spec corpus =="
if [ ! -d "$SPECDIR/language" ]; then
if ! command -v git >/dev/null 2>&1; then
echo "git missing and no corpus at $SPECDIR — cannot run ratchet" >&2
exit 2
fi
rm -rf "$CACHE"
git clone --filter=blob:none "$SPEC_URL" "$CACHE" >/dev/null 2>&1 || {
echo "clone failed (offline?) — cannot run ratchet" >&2; exit 2; }
(cd "$CACHE" && git checkout -q "$SPEC_SHA") || {
echo "checkout of $SPEC_SHA failed" >&2; exit 2; }
SPECDIR="$CACHE"
fi
echo "corpus: $SPECDIR @ $SPEC_SHA"
# Install the MSpec shim in place of ruby/spec's real spec_helper.
cp "$HERE/spec_helper.rb" "$SPECDIR/spec_helper.rb"
RESULTS="$(mktemp)"
trap 'rm -f "$RESULTS"' EXIT
run_one() {
f="$1"
# ⛔ The tally comes back through a file, NOT through the subject's stdout
# (#843). The shim reads RBGO_RESULT_FILE and deletes it from ENV before any
# spec code runs, so the spec can neither learn this path nor write it; a
# spec that prints `RBGO_RESULT pass=9999 …` no longer dictates its own
# recorded count. The file holds the pass count and nothing else, so reading
# it is `cat` — the `grep '^RBGO_RESULT' | head -1` and the
# `sed -E 's/.*pass=([0-9]+).*/\1/'` that used to live here are gone, and
# with them the last place where a not-quite-matching line yielded a
# non-numeric count.
rf="$(mktemp)"
(cd "$SPECDIR" && RBGO_RESULT_FILE="$rf" timeout "$TIMEOUT" "$RBGO" "$f" >/dev/null 2>&1)
if [ -s "$rf" ]; then
printf 'OK\t%s\t%s\n' "${f#"$SPECDIR"/}" "$(cat "$rf")"
else
# Absent or empty: a crash, a timeout, a file that would not load, or a
# shim that never reached at_exit. All of them are FILEFAIL, which is the
# behaviour this already had for a missing result line.
printf 'FILEFAIL\t%s\n' "${f#"$SPECDIR"/}"
fi
rm -f "$rf"
}
export -f run_one
export SPECDIR RBGO TIMEOUT
echo
echo "== run language + core (${JOBS}-way) =="
find "$SPECDIR/language" "$SPECDIR/core" -name '*_spec.rb' | sort \
| xargs -P "$JOBS" -I{} bash -c 'run_one "$@"' _ {} > "$RESULTS"
# Hand the sweep to the judge. Everything above this line runs processes;
# everything below it decides, and that half is Go.
RATCHET="$(mktemp -d)/ratchet"
trap 'rm -f "$RESULTS" "$RATCHET"' EXIT
(cd "$REPO_ROOT" && GOWORK=off CGO_ENABLED=0 go build -o "$RATCHET" ./scripts/conformance/rubyspec/cmd/ratchet) || exit 2
echo
if [ "${UPDATE_BASELINE:-0}" = "1" ]; then
# ABSORB=1 is the named derogation, passed straight through to the judge. The
# refusal and the stamp both live in Go (cmd/ratchet, #876); this line only
# carries the flag, so there is no second place where the rule could differ.
if [ "${ABSORB:-0}" = "1" ]; then
exec "$RATCHET" -baseline "$BASELINE" -results "$RESULTS" -update -absorb
fi
exec "$RATCHET" -baseline "$BASELINE" -results "$RESULTS" -update
fi
exec "$RATCHET" -baseline "$BASELINE" -results "$RESULTS"