From b1d666a88d151daec794ecc3679fe319d218028c Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 12 Nov 2026 10:43:48 +0100 Subject: [PATCH 1/3] rbtest: -perf.commit must equal the measured tree's HEAD Refuse a -perf.commit that is not the exact 40-character HEAD of , or a dirty tree (bench/go.mod and bench/go.sum excepted). Without the flag, record HEAD, suffixed -dirty when the tree is. No git: refuse only if the flag was given. Refs #12 Co-Authored-By: Claude Sonnet 5.5 --- README.md | 5 +++ rbtest/commit.go | 89 +++++++++++++++++++++++++++++++++++++++++++ rbtest/commit_test.go | 82 +++++++++++++++++++++++++++++++++++++++ rbtest/rbtest.go | 7 +++- 4 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 rbtest/commit.go create mode 100644 rbtest/commit_test.go diff --git a/README.md b/README.md index 9dac3d6..8e339bc 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,11 @@ deferred row). Without `-perf.run` nothing is timed and the test passes quietly. Other flags: `-perf.rounds`, `-perf.mintime`, `-perf.maxload`, `-perf.commit`, `-perf.machine`. +| flag | meaning | +|---|---| +| `-perf.commit ` | optional; must equal the full 40-character `git rev-parse HEAD` of ``, and the tree must be clean (a dirty `bench/go.mod` or `bench/go.sum` alone is allowed). Anything else refuses, naming both values or the dirty paths. Without git the flag refuses. | +| (flag omitted) | the commit is filled in from HEAD, suffixed `-dirty` when the tree is; with no git it stays empty | + Nothing is written unless `/go.mod` (the repository's `..` from a `bench/` module) declares the parent of the running binary's module path, or the module itself: a binary of one repository run from another's `bench/` directory refuses, diff --git a/rbtest/commit.go b/rbtest/commit.go new file mode 100644 index 0000000..700bf03 --- /dev/null +++ b/rbtest/commit.go @@ -0,0 +1,89 @@ +package rbtest + +import ( + "bytes" + "fmt" + "os/exec" + "path/filepath" + "strings" +) + +// gitState is what git says about the tree being measured. +type gitState struct { + Head string // the 40-character SHA of HEAD + Dirty []string // paths `git status --porcelain` lists +} + +// gitLookup reads the commit and the dirty paths of the tree at root; tests +// replace it. +var gitLookup = func(root string) (gitState, error) { + head, err := gitOut(root, "rev-parse", "HEAD") + if err != nil { + return gitState{}, err + } + status, err := gitOut(root, "status", "--porcelain") + if err != nil { + return gitState{}, err + } + st := gitState{Head: strings.TrimSpace(head)} + for _, line := range strings.Split(status, "\n") { + if len(line) > 3 { + st.Dirty = append(st.Dirty, strings.TrimSpace(line[3:])) + } + } + return st, nil +} + +func gitOut(root string, args ...string) (string, error) { + cmd := exec.Command("git", append([]string{"-C", root}, args...)...) + var out, errb bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errb + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(errb.String())) + } + return out.String(), nil +} + +// harnessFile reports whether a dirty path is the bench module's go.mod or +// go.sum: consumers bump refbench locally, and those files are the harness, +// not the library. +func harnessFile(p string) bool { + switch filepath.ToSlash(p) { + case "bench/go.mod", "bench/go.sum": + return true + } + return false +} + +// resolveCommit returns the commit to record. With flag set, it must equal +// the 40-character HEAD of the tree at root and the tree must be clean +// (outside the harness files). Without it, HEAD is filled in, suffixed +// "-dirty" when the tree is; with no git and no flag, the commit stays empty. +func resolveCommit(root, flag string) (string, error) { + st, err := gitLookup(root) + if err != nil { + if flag != "" { + return "", fmt.Errorf("-perf.commit %q cannot be checked: %w", flag, err) + } + return "", nil + } + var dirty []string + for _, p := range st.Dirty { + if !harnessFile(p) { + dirty = append(dirty, p) + } + } + if flag != "" { + if flag != st.Head { + return "", fmt.Errorf("-perf.commit is %q, but HEAD of %s is %q: give the full 40-character SHA of the tree measured (git rev-parse HEAD), or omit the flag", flag, root, st.Head) + } + if len(dirty) > 0 { + return "", fmt.Errorf("-perf.commit %q given but the tree at %s is dirty, so a table would claim a commit it did not measure; dirty paths: %s", flag, root, strings.Join(dirty, ", ")) + } + return flag, nil + } + if len(dirty) > 0 { + return st.Head + "-dirty", nil + } + return st.Head, nil +} diff --git a/rbtest/commit_test.go b/rbtest/commit_test.go new file mode 100644 index 0000000..bc287af --- /dev/null +++ b/rbtest/commit_test.go @@ -0,0 +1,82 @@ +package rbtest + +import ( + "errors" + "strings" + "testing" + + "github.com/go-scikit/refbench" +) + +const head = "0123456789abcdef0123456789abcdef01234567" + +func fakeGit(t *testing.T, st gitState, err error) { + t.Helper() + old := gitLookup + t.Cleanup(func() { gitLookup = old }) + gitLookup = func(string) (gitState, error) { return st, err } +} + +func TestResolveCommit(t *testing.T) { + noGit := errors.New("git: executable file not found") + cases := []struct { + name string + st gitState + gitErr error + flag string + want string + wantErr []string // substrings; nil = success + }{ + {name: "match", st: gitState{Head: head}, flag: head, want: head}, + {name: "mismatch", st: gitState{Head: head}, flag: strings.Repeat("a", 40), wantErr: []string{head, strings.Repeat("a", 40)}}, + {name: "prefix only", st: gitState{Head: head}, flag: head[:7], wantErr: []string{head, head[:7]}}, + {name: "dirty library file with flag", st: gitState{Head: head, Dirty: []string{"lib.go", "bench/go.mod"}}, flag: head, wantErr: []string{"lib.go"}}, + {name: "dirty harness only with flag", st: gitState{Head: head, Dirty: []string{"bench/go.mod", "bench/go.sum"}}, flag: head, want: head}, + {name: "no flag clean", st: gitState{Head: head}, want: head}, + {name: "no flag dirty", st: gitState{Head: head, Dirty: []string{"lib.go"}}, want: head + "-dirty"}, + {name: "no flag dirty harness only", st: gitState{Head: head, Dirty: []string{"bench/go.sum"}}, want: head}, + {name: "no git, flag", gitErr: noGit, flag: head, wantErr: []string{"cannot be checked"}}, + {name: "no git, no flag", gitErr: noGit, want: ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + fakeGit(t, c.st, c.gitErr) + got, err := resolveCommit("root", c.flag) + if c.wantErr != nil { + if err == nil { + t.Fatalf("want refusal, got %q", got) + } + for _, s := range c.wantErr { + if !strings.Contains(err.Error(), s) { + t.Errorf("error %q lacks %q", err, s) + } + } + return + } + if err != nil || got != c.want { + t.Fatalf("got %q, %v; want %q", got, err, c.want) + } + }) + } +} + +func TestResolveCommitRealGit(t *testing.T) { + // The default lookup against this very repository, or skipped without git. + st, err := gitLookup(".") + if err != nil { + t.Skipf("no git here: %v", err) + } + if len(st.Head) != 40 { + t.Fatalf("HEAD %q is not a 40-character SHA", st.Head) + } +} + +func TestPerfRefusesWrongCommit(t *testing.T) { + setFlags(t, true, false) + fakeGit(t, gitState{Head: head}, nil) + *commitFlag = head[:7] + got := outcome(func() { perf(&fakeT{}, repo(t, nil), refbench.Options{}) }) + if !strings.HasPrefix(got, "fatal: -perf.commit") || !strings.Contains(got, head) { + t.Fatalf("got %q", got) + } +} diff --git a/rbtest/rbtest.go b/rbtest/rbtest.go index 21fe37e..771b514 100644 --- a/rbtest/rbtest.go +++ b/rbtest/rbtest.go @@ -55,7 +55,7 @@ var ( maxLoadFlag = flag.Float64("perf.maxload", refbench.DefaultMaxLoad, "the 1-minute load gate") onLoadFlag = flag.String("perf.onload", refbench.OnLoadRefuse, "wait or refuse when the load is at or above the gate") waitFlag = flag.Duration("perf.wait", 30*time.Minute, "how long to wait for the gate with -perf.onload=wait") - commitFlag = flag.String("perf.commit", "", "commit recorded in the report") + commitFlag = flag.String("perf.commit", "", "commit recorded in the report: must equal the 40-character HEAD of a clean tree (default: HEAD, marked -dirty when the tree is)") machineFlag = flag.String("perf.machine", "", "machine description recorded in the report (default: GOOS/GOARCH, CPUs)") ) @@ -156,6 +156,11 @@ func perf(t T, cfg refbench.CIConfig, opt refbench.Options) { if err := checkRoot(cfg); err != nil { t.Fatalf("%v", err) } + c, err := resolveCommit(cfg.Root, *commitFlag) + if err != nil { + t.Fatalf("%v", err) + } + opt.Commit = c } switch { case *runFlag: From a4e28a4eb868be0c52e4fb9e3461cbb1b29e1332 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 12 Nov 2026 10:49:24 +0100 Subject: [PATCH 2/3] rbtest: harden the -perf.commit check after review Real-git tests for the status parser (dirty, untracked, rename, xbench, nested module); bench/go.mod exempt only for requires of other modules with the library's replace unchanged, go.sum only alongside it; -perf.record refuses a different commit and records the checked one; dirty marker carries a digest of the changes; untracked-files=all, ignore-submodules=none, GIT_* cleared, 30 s timeout, paths mapped under the module root. Refs #12 Co-Authored-By: Claude Sonnet 5.5 --- README.md | 4 +- rbtest/commit.go | 243 ++++++++++++++++++++++++++++++++++++------ rbtest/commit_test.go | 216 +++++++++++++++++++++++++++++-------- rbtest/rbtest.go | 3 + resume.go | 1 + 5 files changed, 390 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index 8e339bc..9b87a99 100644 --- a/README.md +++ b/README.md @@ -140,8 +140,8 @@ quietly. Other flags: `-perf.rounds`, `-perf.mintime`, `-perf.maxload`, | flag | meaning | |---|---| -| `-perf.commit ` | optional; must equal the full 40-character `git rev-parse HEAD` of ``, and the tree must be clean (a dirty `bench/go.mod` or `bench/go.sum` alone is allowed). Anything else refuses, naming both values or the dirty paths. Without git the flag refuses. | -| (flag omitted) | the commit is filled in from HEAD, suffixed `-dirty` when the tree is; with no git it stays empty | +| `-perf.commit ` | optional; must equal the full 40-character `git rev-parse HEAD` of ``, and the tree must be clean (`bench/go.mod` may differ only in `require` lines of other modules, with the library's `replace` unchanged; `bench/go.sum` only alongside it). Anything else refuses, naming both values or the dirty paths. Without git the flag refuses. | +| (flag omitted) | the commit is filled in from HEAD, suffixed `-dirty-<8 hex digest of the changes>` when the tree is, so `-perf.resume` and `-perf.record` refuse another tree; with no git it stays empty | Nothing is written unless `/go.mod` (the repository's `..` from a `bench/` module) declares the parent of the running binary's module path, or the module diff --git a/rbtest/commit.go b/rbtest/commit.go index 700bf03..f12e382 100644 --- a/rbtest/commit.go +++ b/rbtest/commit.go @@ -2,40 +2,129 @@ package rbtest import ( "bytes" + "context" + "crypto/sha256" + "encoding/hex" "fmt" + "os" "os/exec" "path/filepath" "strings" + "time" ) -// gitState is what git says about the tree being measured. +const ( + harnessMod = "bench/go.mod" + harnessSum = "bench/go.sum" + gitTimeout = 30 * time.Second +) + +// gitState is what git says about the tree being measured. Dirty paths are +// relative to the module root. type gitState struct { - Head string // the 40-character SHA of HEAD - Dirty []string // paths `git status --porcelain` lists + Head string // the 40-character SHA of HEAD + Dirty []string // paths git reports as changed, renamed, deleted or untracked + GoModDiff string // `git diff HEAD -U1000000 -- bench/go.mod` + HashAll string // digest of every change against HEAD (tracked diff and untracked files) + HashLib string // the same without bench/go.mod and bench/go.sum } -// gitLookup reads the commit and the dirty paths of the tree at root; tests +// gitLookup reads the commit and the changes of the tree at root; tests // replace it. -var gitLookup = func(root string) (gitState, error) { +var gitLookup = realGitLookup + +func realGitLookup(root string) (gitState, error) { head, err := gitOut(root, "rev-parse", "HEAD") if err != nil { return gitState{}, err } - status, err := gitOut(root, "status", "--porcelain") + prefix, err := gitOut(root, "rev-parse", "--show-prefix") + if err != nil { + return gitState{}, err + } + prefix = strings.TrimSpace(prefix) + status, err := gitOut(root, "-c", "status.showUntrackedFiles=all", "status", "--porcelain", "-z", "--untracked-files=all", "--ignore-submodules=none") if err != nil { return gitState{}, err } st := gitState{Head: strings.TrimSpace(head)} - for _, line := range strings.Split(status, "\n") { - if len(line) > 3 { - st.Dirty = append(st.Dirty, strings.TrimSpace(line[3:])) + var untracked []string + entries := strings.Split(status, "\x00") + for i := 0; i < len(entries); i++ { + e := entries[i] + if len(e) < 4 { + continue } + xy, p := e[:2], e[3:] + st.Dirty = append(st.Dirty, underRoot(prefix, p)) + if xy == "??" { + untracked = append(untracked, p) + } + if (xy[0] == 'R' || xy[0] == 'C' || xy[1] == 'R' || xy[1] == 'C') && i+1 < len(entries) { + i++ + st.Dirty = append(st.Dirty, underRoot(prefix, entries[i])) + } + } + if len(st.Dirty) == 0 { + return st, nil + } + if st.GoModDiff, err = gitOut(root, "diff", "HEAD", "-U1000000", "--", harnessMod); err != nil { + return gitState{}, err + } + hash := func(exclude bool) (string, error) { + args := []string{"diff", "HEAD", "--binary", "--ignore-submodules=none", "--", "."} + if exclude { + args = append(args, ":(exclude)"+harnessMod, ":(exclude)"+harnessSum) + } + d, err := gitOut(root, args...) + if err != nil { + return "", err + } + h := sha256.New() + h.Write([]byte(d)) + for _, u := range untracked { + rel := underRoot(prefix, u) + if exclude && (rel == harnessMod || rel == harnessSum) { + continue + } + b, _ := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) + fmt.Fprintf(h, "\x00%s\x00", rel) + h.Write(b) + } + return hex.EncodeToString(h.Sum(nil)), nil + } + if st.HashAll, err = hash(false); err != nil { + return gitState{}, err + } + if st.HashLib, err = hash(true); err != nil { + return gitState{}, err } return st, nil } +// underRoot maps a path relative to the repository's top to one relative to +// the module root (prefix is `git rev-parse --show-prefix`, "" or "sub/"); +// a path outside the root is kept as git reports it. +func underRoot(prefix, p string) string { + if prefix != "" { + if rest, ok := strings.CutPrefix(p, prefix); ok { + return rest + } + } + return p +} + func gitOut(root string, args ...string) (string, error) { - cmd := exec.Command("git", append([]string{"-C", root}, args...)...) + ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", root}, args...)...) + for _, kv := range os.Environ() { + switch k, _, _ := strings.Cut(kv, "="); k { + case "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR": + default: + cmd.Env = append(cmd.Env, kv) + } + } var out, errb bytes.Buffer cmd.Stdout, cmd.Stderr = &out, &errb if err := cmd.Run(); err != nil { @@ -44,21 +133,85 @@ func gitOut(root string, args ...string) (string, error) { return out.String(), nil } -// harnessFile reports whether a dirty path is the bench module's go.mod or -// go.sum: consumers bump refbench locally, and those files are the harness, -// not the library. -func harnessFile(p string) bool { - switch filepath.ToSlash(p) { - case "bench/go.mod", "bench/go.sum": - return true +// goModAccepted reports whether a diff of bench/go.mod against HEAD (with +// full context) changes nothing but `require` lines of modules other than +// lib, and keeps the `replace lib => ...` line present and unchanged: a +// changed replace or a pinned require of the library would measure other +// code under a clean HEAD. +func goModAccepted(diff, lib string) bool { + state, replaced, changed := "", false, false + for _, line := range strings.Split(diff, "\n") { + if line == "" || strings.HasPrefix(line, "diff ") || strings.HasPrefix(line, "index ") || + strings.HasPrefix(line, "--- ") || strings.HasPrefix(line, "+++ ") || strings.HasPrefix(line, "@@") || + strings.HasPrefix(line, "\\") { + continue + } + mark, body := line[0], line[1:] + if i := strings.Index(body, "//"); i >= 0 { + body = body[:i] + } + f := strings.Fields(body) + if len(f) == 0 { + continue + } + if mark != ' ' && mark != '+' && mark != '-' { + return false + } + isChange := mark != ' ' + changed = changed || isChange + switch { + case len(f) == 2 && f[1] == "(": + state = f[0] + if isChange { + return false + } + case f[0] == ")": + state = "" + if isChange { + return false + } + case f[0] == "require" && len(f) >= 3: + if isChange && f[1] == lib { + return false + } + case state == "require": + if isChange && f[0] == lib { + return false + } + case f[0] == "replace" && len(f) >= 3 && f[1] == lib && hasArrow(f): + if isChange { + return false + } + replaced = true + case state == "replace" && f[0] == lib && hasArrow(f): + if isChange { + return false + } + replaced = true + default: + if isChange { + return false + } + } + } + return changed && replaced +} + +func hasArrow(f []string) bool { + for _, s := range f { + if s == "=>" { + return true + } } return false } // resolveCommit returns the commit to record. With flag set, it must equal -// the 40-character HEAD of the tree at root and the tree must be clean -// (outside the harness files). Without it, HEAD is filled in, suffixed -// "-dirty" when the tree is; with no git and no flag, the commit stays empty. +// the 40-character HEAD of the tree at root and the tree must be clean, +// except for bench/go.mod and bench/go.sum when go.mod changes only requires +// of other modules (see goModAccepted). Without the flag, HEAD is filled in, +// suffixed "-dirty-<8 hex of the changes' digest>" when the tree is dirty; +// with no git and no flag, the commit stays empty. func resolveCommit(root, flag string) (string, error) { st, err := gitLookup(root) if err != nil { @@ -67,23 +220,53 @@ func resolveCommit(root, flag string) (string, error) { } return "", nil } + if flag != "" && flag != st.Head { + return "", fmt.Errorf("-perf.commit is %q, but HEAD of %s is %q: give the full 40-character SHA of the tree measured (git rev-parse HEAD), or omit the flag", flag, root, st.Head) + } + if len(st.Dirty) == 0 { + return st.Head, nil + } + harnessOK := false var dirty []string + var goModDirty, sumDirty bool for _, p := range st.Dirty { - if !harnessFile(p) { + switch s := filepath.ToSlash(p); { + case s == harnessMod: + goModDirty = true + case s == harnessSum: + sumDirty = true + default: dirty = append(dirty, p) } } + if goModDirty { + lib, err := modulePath(filepath.Join(root, "go.mod")) + harnessOK = err == nil && goModAccepted(st.GoModDiff, lib) + } + switch { + case goModDirty && !harnessOK: + dirty = append(dirty, harnessMod+" (more than requires of other modules changed, or the replace of the library moved)") + case sumDirty && !goModDirty: + dirty = append(dirty, harnessSum+" (without an accepted "+harnessMod+")") + } + if len(dirty) == 0 && !goModDirty && !sumDirty { + return st.Head, nil + } if flag != "" { - if flag != st.Head { - return "", fmt.Errorf("-perf.commit is %q, but HEAD of %s is %q: give the full 40-character SHA of the tree measured (git rev-parse HEAD), or omit the flag", flag, root, st.Head) - } - if len(dirty) > 0 { - return "", fmt.Errorf("-perf.commit %q given but the tree at %s is dirty, so a table would claim a commit it did not measure; dirty paths: %s", flag, root, strings.Join(dirty, ", ")) + if len(dirty) == 0 { + return flag, nil } - return flag, nil + return "", fmt.Errorf("-perf.commit %q given but the tree at %s is dirty, so a table would claim a commit it did not measure; dirty paths: %s", flag, root, strings.Join(dirty, ", ")) + } + if len(dirty) == 0 { + return st.Head, nil // only an accepted harness change + } + h := st.HashAll + if harnessOK { + h = st.HashLib } - if len(dirty) > 0 { - return st.Head + "-dirty", nil + if len(h) > 8 { + h = h[:8] } - return st.Head, nil + return st.Head + "-dirty-" + h, nil } diff --git a/rbtest/commit_test.go b/rbtest/commit_test.go index bc287af..239d255 100644 --- a/rbtest/commit_test.go +++ b/rbtest/commit_test.go @@ -1,82 +1,208 @@ package rbtest import ( - "errors" + "os" + "os/exec" + "path/filepath" "strings" "testing" "github.com/go-scikit/refbench" ) -const head = "0123456789abcdef0123456789abcdef01234567" +const fakeHead = "0123456789abcdef0123456789abcdef01234567" -func fakeGit(t *testing.T, st gitState, err error) { +func needGit(t *testing.T) { t.Helper() - old := gitLookup - t.Cleanup(func() { gitLookup = old }) - gitLookup = func(string) (gitState, error) { return st, err } + if _, err := exec.LookPath("git"); err != nil { + t.Skip("no git") + } +} + +func put(t *testing.T, dir, rel, content string) { + t.Helper() + p := filepath.Join(dir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func git(t *testing.T, dir string, args ...string) string { + t.Helper() + out, err := gitOut(dir, append([]string{"-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"}, args...)...) + if err != nil { + t.Fatal(err) + } + return out } -func TestResolveCommit(t *testing.T) { - noGit := errors.New("git: executable file not found") +const benchMod = `module example.com/lib/bench + +go 1.27 + +require ( + example.com/lib v0.0.0 + example.com/other v1.0.0 +) + +replace example.com/lib => ../ +` + +// repoAt makes a git repository whose module root is dir/sub ("" = the top). +func repoAt(t *testing.T, sub string) (top, root, head string) { + t.Helper() + needGit(t) + top = t.TempDir() + root = filepath.Join(top, sub) + put(t, root, "go.mod", "module example.com/lib\n\ngo 1.27\n") + put(t, root, "lib.go", "package lib\n") + put(t, root, "bench/go.mod", benchMod) + put(t, root, "bench/go.sum", "a b c\n") + put(t, root, "xbench/go.mod", benchMod) + git(t, top, "init", "-q") + git(t, top, "add", "-A") + git(t, top, "commit", "-q", "-m", "init") + return top, root, strings.TrimSpace(git(t, top, "rev-parse", "HEAD")) +} + +func TestResolveCommitRealGit(t *testing.T) { + bump := strings.Replace(benchMod, "example.com/other v1.0.0", "example.com/other v1.1.0", 1) cases := []struct { name string - st gitState - gitErr error - flag string - want string - wantErr []string // substrings; nil = success + sub string + mutate func(t *testing.T, top, root string) + wantErr string // substring; "" = accepted }{ - {name: "match", st: gitState{Head: head}, flag: head, want: head}, - {name: "mismatch", st: gitState{Head: head}, flag: strings.Repeat("a", 40), wantErr: []string{head, strings.Repeat("a", 40)}}, - {name: "prefix only", st: gitState{Head: head}, flag: head[:7], wantErr: []string{head, head[:7]}}, - {name: "dirty library file with flag", st: gitState{Head: head, Dirty: []string{"lib.go", "bench/go.mod"}}, flag: head, wantErr: []string{"lib.go"}}, - {name: "dirty harness only with flag", st: gitState{Head: head, Dirty: []string{"bench/go.mod", "bench/go.sum"}}, flag: head, want: head}, - {name: "no flag clean", st: gitState{Head: head}, want: head}, - {name: "no flag dirty", st: gitState{Head: head, Dirty: []string{"lib.go"}}, want: head + "-dirty"}, - {name: "no flag dirty harness only", st: gitState{Head: head, Dirty: []string{"bench/go.sum"}}, want: head}, - {name: "no git, flag", gitErr: noGit, flag: head, wantErr: []string{"cannot be checked"}}, - {name: "no git, no flag", gitErr: noGit, want: ""}, + {name: "clean", mutate: func(*testing.T, string, string) {}}, + {name: "dirty library file", mutate: func(t *testing.T, _, r string) { put(t, r, "lib.go", "package lib // x\n") }, wantErr: "lib.go"}, + {name: "untracked go file", mutate: func(t *testing.T, _, r string) { put(t, r, "new.go", "package lib\n") }, wantErr: "new.go"}, + {name: "rename", mutate: func(t *testing.T, top, r string) { git(t, r, "mv", "lib.go", "lib2.go") }, wantErr: "lib2.go"}, + {name: "xbench/go.mod is not the harness", mutate: func(t *testing.T, _, r string) { put(t, r, "xbench/go.mod", bump) }, wantErr: "xbench/go.mod"}, + {name: "bench/go.mod require bump", mutate: func(t *testing.T, _, r string) { put(t, r, "bench/go.mod", bump) }}, + {name: "bench/go.mod and go.sum bump", mutate: func(t *testing.T, _, r string) { + put(t, r, "bench/go.mod", bump) + put(t, r, "bench/go.sum", "a b c\nd e f\n") + }}, + {name: "bench/go.sum alone", mutate: func(t *testing.T, _, r string) { put(t, r, "bench/go.sum", "x\n") }, wantErr: "bench/go.sum"}, + {name: "replace of the library changed", mutate: func(t *testing.T, _, r string) { + put(t, r, "bench/go.mod", strings.Replace(benchMod, "=> ../", "=> ../elsewhere", 1)) + }, wantErr: "bench/go.mod"}, + {name: "replace of the library removed", mutate: func(t *testing.T, _, r string) { + put(t, r, "bench/go.mod", strings.Replace(benchMod, "replace example.com/lib => ../\n", "", 1)) + }, wantErr: "bench/go.mod"}, + {name: "require of the library pinned", mutate: func(t *testing.T, _, r string) { + put(t, r, "bench/go.mod", strings.Replace(benchMod, "example.com/lib v0.0.0", "example.com/lib v0.9.0", 1)) + }, wantErr: "bench/go.mod"}, + {name: "go directive changed", mutate: func(t *testing.T, _, r string) { + put(t, r, "bench/go.mod", strings.Replace(benchMod, "go 1.27", "go 1.26", 1)) + }, wantErr: "bench/go.mod"}, + {name: "nested module, harness bump", sub: "sub", mutate: func(t *testing.T, _, r string) { put(t, r, "bench/go.mod", bump) }}, + {name: "nested module, library file", sub: "sub", mutate: func(t *testing.T, _, r string) { put(t, r, "lib.go", "package lib // x\n") }, wantErr: "lib.go"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - fakeGit(t, c.st, c.gitErr) - got, err := resolveCommit("root", c.flag) - if c.wantErr != nil { - if err == nil { - t.Fatalf("want refusal, got %q", got) - } - for _, s := range c.wantErr { - if !strings.Contains(err.Error(), s) { - t.Errorf("error %q lacks %q", err, s) - } + top, root, head := repoAt(t, c.sub) + c.mutate(t, top, root) + got, err := resolveCommit(root, head) + if c.wantErr == "" { + if err != nil || got != head { + t.Fatalf("got %q, %v; want %q accepted", got, err, head) } return } - if err != nil || got != c.want { - t.Fatalf("got %q, %v; want %q", got, err, c.want) + if err == nil || !strings.Contains(err.Error(), c.wantErr) { + t.Fatalf("got %q, %v; want a refusal naming %q", got, err, c.wantErr) } }) } } -func TestResolveCommitRealGit(t *testing.T) { - // The default lookup against this very repository, or skipped without git. - st, err := gitLookup(".") - if err != nil { - t.Skipf("no git here: %v", err) +func TestResolveCommitMismatchAndPrefix(t *testing.T) { + _, root, head := repoAt(t, "") + for _, flag := range []string{head[:7], strings.Repeat("a", 40)} { + _, err := resolveCommit(root, flag) + if err == nil || !strings.Contains(err.Error(), head) || !strings.Contains(err.Error(), flag) { + t.Errorf("flag %q: %v", flag, err) + } } - if len(st.Head) != 40 { - t.Fatalf("HEAD %q is not a 40-character SHA", st.Head) +} + +func TestDirtyMarkerTellsTreesApart(t *testing.T) { + _, root, head := repoAt(t, "") + mark := func(content string) string { + put(t, root, "lib.go", content) + got, err := resolveCommit(root, "") + if err != nil { + t.Fatal(err) + } + return got + } + a, b, a2 := mark("package lib // a\n"), mark("package lib // b\n"), mark("package lib // a\n") + if !strings.HasPrefix(a, head+"-dirty-") || len(a) != len(head)+len("-dirty-")+8 { + t.Fatalf("marker %q", a) + } + if a == b || a != a2 { + t.Fatalf("markers a=%q b=%q a2=%q", a, b, a2) + } + put(t, root, "lib.go", "package lib\n") + if got, _ := resolveCommit(root, ""); got != head { + t.Fatalf("clean again: %q", got) + } +} + +func TestResolveCommitNoGit(t *testing.T) { + old := gitLookup + t.Cleanup(func() { gitLookup = old }) + gitLookup = func(string) (gitState, error) { return gitState{}, os.ErrNotExist } + if _, err := resolveCommit("r", fakeHead); err == nil || !strings.Contains(err.Error(), "cannot be checked") { + t.Fatalf("flag without git: %v", err) + } + if got, err := resolveCommit("r", ""); err != nil || got != "" { + t.Fatalf("no flag, no git: %q, %v", got, err) + } +} + +func TestGitEnvIsCleared(t *testing.T) { + _, root, head := repoAt(t, "") + t.Setenv("GIT_DIR", filepath.Join(t.TempDir(), "nowhere")) + t.Setenv("GIT_WORK_TREE", t.TempDir()) + if got, err := resolveCommit(root, head); err != nil || got != head { + t.Fatalf("got %q, %v", got, err) } } func TestPerfRefusesWrongCommit(t *testing.T) { setFlags(t, true, false) - fakeGit(t, gitState{Head: head}, nil) - *commitFlag = head[:7] + old := gitLookup + t.Cleanup(func() { gitLookup = old }) + gitLookup = func(string) (gitState, error) { return gitState{Head: fakeHead}, nil } + *commitFlag = fakeHead[:7] got := outcome(func() { perf(&fakeT{}, repo(t, nil), refbench.Options{}) }) - if !strings.HasPrefix(got, "fatal: -perf.commit") || !strings.Contains(got, head) { + if !strings.HasPrefix(got, "fatal: -perf.commit") || !strings.Contains(got, fakeHead) { + t.Fatalf("got %q", got) + } +} + +func TestRecordRefusesOtherCommit(t *testing.T) { + setFlags(t, false, true) + old := gitLookup + t.Cleanup(func() { gitLookup = old }) + gitLookup = func(string) (gitState, error) { return gitState{Head: fakeHead}, nil } + cfg := repo(t, nil) + jsonPath, _ := cfg.Paths() + prev := &refbench.Report{Schema: refbench.SchemaVersion, Status: refbench.StatusMeasured, Commit: strings.Repeat("b", 40), Repo: "r/x"} + var b strings.Builder + if err := prev.Write(&b); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jsonPath, []byte(b.String()), 0o644); err != nil { + t.Fatal(err) + } + got := outcome(func() { perf(&fakeT{}, cfg, refbench.Options{}) }) + if !strings.Contains(got, "mix commits") { t.Fatalf("got %q", got) } } diff --git a/rbtest/rbtest.go b/rbtest/rbtest.go index 771b514..915fca6 100644 --- a/rbtest/rbtest.go +++ b/rbtest/rbtest.go @@ -213,6 +213,9 @@ func perf(t T, cfg refbench.CIConfig, opt refbench.Options) { if err != nil { t.Fatalf("%v", err) } + if prev != nil && prev.Status == refbench.StatusMeasured && prev.Commit != opt.Commit { + t.Fatalf("-perf.record: perf.json was measured at commit %q, this run is at %q: recording allocation counts onto it would mix commits (run -perf.run again)", prev.Commit, opt.Commit) + } write(t, cfg, refbench.MergeAllocs(prev, rep)) default: if err := refbench.RunCI(cfg); err != nil { diff --git a/resume.go b/resume.go index f2ea23e..32777b2 100644 --- a/resume.go +++ b/resume.go @@ -105,6 +105,7 @@ func MergeAllocs(prev, rec *Report) *Report { old[r.Name] = r } out := *prev + out.Commit = rec.Commit // the allocation counts are those of this commit out.Rows = make([]Row, 0, len(rec.Rows)) for _, r := range rec.Rows { if o, ok := old[r.Name]; ok { From be61ca4d420c417e7d6b148723174fc37f8f988b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 12 Nov 2026 10:53:35 +0100 Subject: [PATCH 3/3] rbtest: reach the 100% coverage gate Fewer error branches (one batched git call), a goModAccepted table, hash-choice and outside-git tests. Refs #12 Co-Authored-By: Claude Sonnet 5.5 --- rbtest/commit.go | 164 ++++++++++++++++-------------------------- rbtest/commit_test.go | 80 +++++++++++++++++++++ 2 files changed, 142 insertions(+), 102 deletions(-) diff --git a/rbtest/commit.go b/rbtest/commit.go index f12e382..f4fa0bb 100644 --- a/rbtest/commit.go +++ b/rbtest/commit.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "time" ) @@ -34,22 +35,22 @@ type gitState struct { var gitLookup = realGitLookup func realGitLookup(root string) (gitState, error) { - head, err := gitOut(root, "rev-parse", "HEAD") + diff := []string{"diff", "HEAD", "--binary", "--ignore-submodules=none", "--", "."} + outs, err := gitAll(root, [][]string{ + {"rev-parse", "HEAD"}, + {"rev-parse", "--show-prefix"}, + {"-c", "status.showUntrackedFiles=all", "status", "--porcelain", "-z", "--untracked-files=all", "--ignore-submodules=none"}, + {"diff", "HEAD", "-U1000000", "--", harnessMod}, + diff, + append(slices.Clone(diff), ":(exclude)"+harnessMod, ":(exclude)"+harnessSum), + }) if err != nil { return gitState{}, err } - prefix, err := gitOut(root, "rev-parse", "--show-prefix") - if err != nil { - return gitState{}, err - } - prefix = strings.TrimSpace(prefix) - status, err := gitOut(root, "-c", "status.showUntrackedFiles=all", "status", "--porcelain", "-z", "--untracked-files=all", "--ignore-submodules=none") - if err != nil { - return gitState{}, err - } - st := gitState{Head: strings.TrimSpace(head)} + prefix := strings.TrimSpace(outs[1]) + st := gitState{Head: strings.TrimSpace(outs[0]), GoModDiff: outs[3]} var untracked []string - entries := strings.Split(status, "\x00") + entries := strings.Split(outs[2], "\x00") for i := 0; i < len(entries); i++ { e := entries[i] if len(e) < 4 { @@ -58,62 +59,56 @@ func realGitLookup(root string) (gitState, error) { xy, p := e[:2], e[3:] st.Dirty = append(st.Dirty, underRoot(prefix, p)) if xy == "??" { - untracked = append(untracked, p) + untracked = append(untracked, underRoot(prefix, p)) } - if (xy[0] == 'R' || xy[0] == 'C' || xy[1] == 'R' || xy[1] == 'C') && i+1 < len(entries) { + if strings.ContainsAny(xy, "RC") && i+1 < len(entries) { i++ st.Dirty = append(st.Dirty, underRoot(prefix, entries[i])) } } - if len(st.Dirty) == 0 { - return st, nil - } - if st.GoModDiff, err = gitOut(root, "diff", "HEAD", "-U1000000", "--", harnessMod); err != nil { - return gitState{}, err - } - hash := func(exclude bool) (string, error) { - args := []string{"diff", "HEAD", "--binary", "--ignore-submodules=none", "--", "."} - if exclude { - args = append(args, ":(exclude)"+harnessMod, ":(exclude)"+harnessSum) - } - d, err := gitOut(root, args...) - if err != nil { - return "", err - } - h := sha256.New() - h.Write([]byte(d)) - for _, u := range untracked { - rel := underRoot(prefix, u) - if exclude && (rel == harnessMod || rel == harnessSum) { - continue - } - b, _ := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) - fmt.Fprintf(h, "\x00%s\x00", rel) - h.Write(b) + st.HashAll = digest(root, outs[4], untracked, false) + st.HashLib = digest(root, outs[5], untracked, true) + return st, nil +} + +// digest hashes a tracked diff and the content of the untracked files. +func digest(root, diff string, untracked []string, skipHarness bool) string { + h := sha256.New() + h.Write([]byte(diff)) + for _, rel := range untracked { + if skipHarness && (rel == harnessMod || rel == harnessSum) { + continue } - return hex.EncodeToString(h.Sum(nil)), nil - } - if st.HashAll, err = hash(false); err != nil { - return gitState{}, err + b, _ := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) + fmt.Fprintf(h, "\x00%s\x00", rel) + h.Write(b) } - if st.HashLib, err = hash(true); err != nil { - return gitState{}, err - } - return st, nil + return hex.EncodeToString(h.Sum(nil)) } // underRoot maps a path relative to the repository's top to one relative to // the module root (prefix is `git rev-parse --show-prefix`, "" or "sub/"); // a path outside the root is kept as git reports it. func underRoot(prefix, p string) string { - if prefix != "" { - if rest, ok := strings.CutPrefix(p, prefix); ok { - return rest - } + if rest, ok := strings.CutPrefix(p, prefix); ok { + return rest } return p } +// gitAll runs each git command in turn and returns the outputs. +func gitAll(root string, cmds [][]string) ([]string, error) { + outs := make([]string, 0, len(cmds)) + for _, args := range cmds { + out, err := gitOut(root, args...) + if err != nil { + return nil, err + } + outs = append(outs, out) + } + return outs, nil +} + func gitOut(root string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), gitTimeout) defer cancel() @@ -146,19 +141,18 @@ func goModAccepted(diff, lib string) bool { strings.HasPrefix(line, "\\") { continue } - mark, body := line[0], line[1:] - if i := strings.Index(body, "//"); i >= 0 { - body = body[:i] + mark := line[:1] + if !strings.Contains(" +-", mark) { + return false // new file, deleted file, rename... } + body, _, _ := strings.Cut(line[1:], "//") f := strings.Fields(body) if len(f) == 0 { continue } - if mark != ' ' && mark != '+' && mark != '-' { - return false - } - isChange := mark != ' ' + isChange := mark != " " changed = changed || isChange + isLibReplace := (f[0] == "replace" && len(f) > 1 && f[1] == lib || state == "replace" && f[0] == lib) && slices.Contains(f, "=>") switch { case len(f) == 2 && f[1] == "(": state = f[0] @@ -170,24 +164,13 @@ func goModAccepted(diff, lib string) bool { if isChange { return false } - case f[0] == "require" && len(f) >= 3: - if isChange && f[1] == lib { - return false - } - case state == "require": - if isChange && f[0] == lib { - return false - } - case f[0] == "replace" && len(f) >= 3 && f[1] == lib && hasArrow(f): - if isChange { - return false - } - replaced = true - case state == "replace" && f[0] == lib && hasArrow(f): + case isLibReplace: if isChange { return false } replaced = true + case f[0] == "require" && len(f) > 2 && f[1] != lib, state == "require" && f[0] != lib: + // a require of another module: a change is accepted default: if isChange { return false @@ -197,15 +180,6 @@ func goModAccepted(diff, lib string) bool { return changed && replaced } -func hasArrow(f []string) bool { - for _, s := range f { - if s == "=>" { - return true - } - } - return false -} - // resolveCommit returns the commit to record. With flag set, it must equal // the 40-character HEAD of the tree at root and the tree must be clean, // except for bench/go.mod and bench/go.sum when go.mod changes only requires @@ -223,12 +197,8 @@ func resolveCommit(root, flag string) (string, error) { if flag != "" && flag != st.Head { return "", fmt.Errorf("-perf.commit is %q, but HEAD of %s is %q: give the full 40-character SHA of the tree measured (git rev-parse HEAD), or omit the flag", flag, root, st.Head) } - if len(st.Dirty) == 0 { - return st.Head, nil - } - harnessOK := false var dirty []string - var goModDirty, sumDirty bool + goModDirty, sumDirty, harnessOK := false, false, false for _, p := range st.Dirty { switch s := filepath.ToSlash(p); { case s == harnessMod: @@ -241,32 +211,22 @@ func resolveCommit(root, flag string) (string, error) { } if goModDirty { lib, err := modulePath(filepath.Join(root, "go.mod")) - harnessOK = err == nil && goModAccepted(st.GoModDiff, lib) + if harnessOK = err == nil && goModAccepted(st.GoModDiff, lib); !harnessOK { + dirty = append(dirty, harnessMod+" (more than requires of other modules changed, or the replace of the library moved)") + } } - switch { - case goModDirty && !harnessOK: - dirty = append(dirty, harnessMod+" (more than requires of other modules changed, or the replace of the library moved)") - case sumDirty && !goModDirty: + if sumDirty && !goModDirty { dirty = append(dirty, harnessSum+" (without an accepted "+harnessMod+")") } - if len(dirty) == 0 && !goModDirty && !sumDirty { + if len(dirty) == 0 { return st.Head, nil } if flag != "" { - if len(dirty) == 0 { - return flag, nil - } return "", fmt.Errorf("-perf.commit %q given but the tree at %s is dirty, so a table would claim a commit it did not measure; dirty paths: %s", flag, root, strings.Join(dirty, ", ")) } - if len(dirty) == 0 { - return st.Head, nil // only an accepted harness change - } h := st.HashAll if harnessOK { h = st.HashLib } - if len(h) > 8 { - h = h[:8] - } - return st.Head + "-dirty-" + h, nil + return st.Head + "-dirty-" + h[:8], nil } diff --git a/rbtest/commit_test.go b/rbtest/commit_test.go index 239d255..51def86 100644 --- a/rbtest/commit_test.go +++ b/rbtest/commit_test.go @@ -206,3 +206,83 @@ func TestRecordRefusesOtherCommit(t *testing.T) { t.Fatalf("got %q", got) } } + +func TestGoModAccepted(t *testing.T) { + const hdr = "diff --git a/bench/go.mod b/bench/go.mod\nindex 1..2 100644\n--- a/bench/go.mod\n+++ b/bench/go.mod\n@@ -1,9 +1,9 @@\n\\ No newline\n\n" + const lib = "example.com/lib" + block := func(req, rep string) string { + return hdr + " module example.com/lib/bench\n\n require (\n" + req + " )\n\n" + rep + } + cases := []struct { + name string + diff string + want bool + }{ + {"require of another module bumped", block("-\texample.com/o v1\n+\texample.com/o v2 // indirect\n", " replace example.com/lib => ../\n"), true}, + {"single-line require of another module", hdr + "-require example.com/o v1\n+require example.com/o v2\n replace example.com/lib => ../\n", true}, + {"replace block kept", hdr + "-require example.com/o v1\n+require example.com/o v2\n replace (\n \texample.com/lib => ../\n )\n", true}, + {"library require pinned", block("-\texample.com/lib v0\n+\texample.com/lib v1\n", " replace example.com/lib => ../\n"), false}, + {"single-line library require", hdr + "-require example.com/lib v0\n+require example.com/lib v1\n replace example.com/lib => ../\n", false}, + {"replace changed", block("+\texample.com/o v2\n", "-replace example.com/lib => ../\n+replace example.com/lib => ../x\n"), false}, + {"replace in a block changed", hdr + "+require example.com/o v2\n replace (\n-\texample.com/lib => ../\n+\texample.com/lib => ../x\n )\n", false}, + {"replace missing", hdr + "+require example.com/o v2\n", false}, + {"nothing changed", hdr + " replace example.com/lib => ../\n", false}, + {"go directive changed", hdr + "-go 1.26\n+go 1.27\n+require example.com/o v2\n replace example.com/lib => ../\n", false}, + {"block opener changed", hdr + "+require (\n replace example.com/lib => ../\n", false}, + {"block closer changed", hdr + " require (\n+)\n replace example.com/lib => ../\n", false}, + {"new file", "new file mode 100644\n+module x\n", false}, + } + for _, c := range cases { + if got := goModAccepted(c.diff, lib); got != c.want { + t.Errorf("%s: got %v, want %v", c.name, got, c.want) + } + } +} + +func TestResolveCommitHashChoice(t *testing.T) { + root := t.TempDir() + put(t, root, "go.mod", "module example.com/lib\n") + old := gitLookup + t.Cleanup(func() { gitLookup = old }) + const diff = "--- a\n+++ b\n+require example.com/o v2\n replace example.com/lib => ../\n" + st := gitState{Head: fakeHead, Dirty: []string{"lib.go", "bench/go.mod"}, GoModDiff: diff, HashAll: "aaaaaaaa11", HashLib: "bbbbbbbb22"} + gitLookup = func(string) (gitState, error) { return st, nil } + if got, err := resolveCommit(root, ""); err != nil || got != fakeHead+"-dirty-bbbbbbbb" { + t.Fatalf("accepted harness: %q, %v", got, err) + } + st.GoModDiff = "+go 1.2\n" + if got, err := resolveCommit(root, ""); err != nil || got != fakeHead+"-dirty-aaaaaaaa" { + t.Fatalf("refused harness: %q, %v", got, err) + } + if err := os.Remove(filepath.Join(root, "go.mod")); err != nil { + t.Fatal(err) + } + st.GoModDiff = diff + if got, err := resolveCommit(root, ""); err != nil || got != fakeHead+"-dirty-aaaaaaaa" { + t.Fatalf("unreadable root go.mod must not exempt the harness: %q, %v", got, err) + } + if underRoot("sub/", "other/x") != "other/x" || underRoot("sub/", "sub/x") != "x" || underRoot("", "x") != "x" { + t.Fatal("underRoot") + } + st.Dirty = []string{"bench/go.mod"} + if _, err := resolveCommit(root, fakeHead); err == nil || !strings.Contains(err.Error(), "bench/go.mod") { + t.Fatalf("flag with an unreadable root go.mod: %v", err) + } +} + +func TestLookupOutsideGit(t *testing.T) { + needGit(t) + if _, err := realGitLookup(t.TempDir()); err == nil { + t.Skip("temporary directory is inside a git repository") + } +} + +func TestUntrackedHarnessFiles(t *testing.T) { + _, root, head := repoAt(t, "") + git(t, root, "rm", "-q", "--cached", "bench/go.sum") + git(t, root, "commit", "-q", "-m", "untrack") + head = strings.TrimSpace(git(t, root, "rev-parse", "HEAD")) + if _, err := resolveCommit(root, head); err == nil || !strings.Contains(err.Error(), "bench/go.sum") { + t.Fatalf("untracked go.sum: %v", err) + } +}