diff --git a/.agents/skills/verify-lane-schedule/SKILL.md b/.agents/skills/verify-lane-schedule/SKILL.md index f5e14ec7..f4f6741b 100644 --- a/.agents/skills/verify-lane-schedule/SKILL.md +++ b/.agents/skills/verify-lane-schedule/SKILL.md @@ -226,7 +226,7 @@ With every fact TRUE in the code, setting each fact FALSE in the model alone fai ## Coverage record -Sync commit: `2765e686322c73921d30261313617e616c769ce4` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). +Sync commit: `38962a11d1b5cf1d1669f937ce7996c70147cfb1` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). Every declaration in these files is covered: diff --git a/.agents/skills/verify-lane-schedule/model/fingerprints.txt b/.agents/skills/verify-lane-schedule/model/fingerprints.txt index eeb50472..34fe2eaa 100644 --- a/.agents/skills/verify-lane-schedule/model/fingerprints.txt +++ b/.agents/skills/verify-lane-schedule/model/fingerprints.txt @@ -142,6 +142,7 @@ internal/events/refstore.go:type_RepoSource 714bac4245f62bc423df8a3c9a5ebaaa8404 internal/events/refstore.go:type_RepositoryRef c3a70bb7531da30f38f6cf40bdea9b1b3a25fb01ef371723c59181d7b8cfc7bc internal/events/refstore.go:var__ 9158a92adb23eeae820deb629856fb0284783c74457ee23a53cf824402ec3fee internal/githttp/hook.go:RunHook f34a6e070b5654470ebb19f9a67a7179501e661a70a4771d74802a89dc55f5a1 +internal/githttp/hook.go:answerDeleteOfAbsentRef 4e4d160ed99aec9e09d5b725718304451b151e8d326708f7b28f48efb1703006 internal/githttp/hook.go:atomicDenied 3fce86bb64fb256a2988c2d2fe12fffc5bc2546fc2b9a9dacb714963e1ca405c internal/githttp/hook.go:atomicPushDFCheck f3610ea748f8a1cddc0f03df8dd643d3100cf806abfd163507406fae261058e0 internal/githttp/hook.go:authorityDoomedCreations db1ff9f71e8ce4b73368ce27843037139e4d60d20f8890175ca5afcfbc43b64b @@ -259,7 +260,7 @@ internal/githttp/hook.go:type_refValuesProber 0389595e3fb341c7e67784d27674d922dd internal/githttp/hook.go:type_reservationClass 48d346b5a457165c93ed2a9aca43bcd39438f05201427fa2351d8b3a4066ff29 internal/githttp/hook.go:type_sigVerifier afbcc825325e58413e4a3646073bab61862ebda41b2c8c9f6fefae2ad35a4745 internal/githttp/hook.go:updateBarrierEachPath 53afd6c803fba2472970b4a81e706f4e6c975c84753c0c94714bd52c1fdd0fcf -internal/githttp/hook.go:updateHook 2d7d924d17781fd711f5c8395f2b0e3d4511ef5e12e4643ef8c8716a607f670f +internal/githttp/hook.go:updateHook 792fc8e3ab51414a4a63665325ba22a4e7b8670e38b95620c973c6740e329361 internal/githttp/hook.go:var_WrapHookRefStore 2a84a08163beaeaef666d54ff198e62e68b8e70a83fa6f46f9025b30c28ce5c5 internal/githttp/hook.go:var__ 14f3ac5e5fb70c7fe12bcffeaa3f5891a5f61b624d170427d94afa8153dcf5fd internal/githttp/hook.go:var__#2 d54a83bcc2c7fb6caf4b5040f8c28ca780368fb5c2c578f05c5a48f6edb6dd97 diff --git a/docs/explanation/git-protocol-and-namespaces.md b/docs/explanation/git-protocol-and-namespaces.md index d9db19cb..9943a949 100644 --- a/docs/explanation/git-protocol-and-namespaces.md +++ b/docs/explanation/git-protocol-and-namespaces.md @@ -43,6 +43,17 @@ The receive path must: nothing, and a push whose objects only refused commands need is declined whole, with no pack stored. A branch delete and a new value that the server does not have are not refused here. Other namespaces, such as tags, take any object. +- Answer a delete of a ref that the client's replica does not list. git sends + `git push :` for such a ref as a command from the zero id to the zero id, + and stock git answers `ok` with the warning `deleting a non-existent ref`. The hooks do + the same when the ref store holds no such ref. When the store holds it (the replica's + cache lags, or the ref was created since the client read the refs), the hooks refuse + that command with a per-ref `ng` (`hook declined`) whose `remote:` text names the ref and + the first 8 hex digits of its SHA (` exists at in the ref store, but this + replica does not list it`), and `serve` logs an operator line. Fetch, then push the + delete again. An `--atomic` push with such a command is refused whole by pre-receive + (` already exists (created concurrently); fetch first`), as for any command whose + old value the store contradicts, with no SHA and no operator line. - Map namespaces: `+ephemeral` remotes see and update only the ephemeral ref space; persistent remotes must not see ephemeral refs (git namespaces or separate ref stores). open.storage gives the `+ephemeral` remote its own storage key. The HTTP API, though, stores its diff --git a/docs/how-to/deploy-production.md b/docs/how-to/deploy-production.md index 0730fa47..11c24ed2 100644 --- a/docs/how-to/deploy-production.md +++ b/docs/how-to/deploy-production.md @@ -379,8 +379,10 @@ Remove any of these refs from the ref store with namespace (for example a notes ref). There, delete the ref that a replica's cache lists, with `git push :` through that replica. That is usually the ref that the push created, not the one that step 3 printed: the restored ref was at a missing - object, so no cache lists it, and the push that deletes it fails. Ask the pusher to push - it again under a free name: the old name conflicts with the restored ref. + object, so no cache lists it, and the replica refuses a push that deletes it (` + exists at in the ref store, but this replica does not list it`, and `serve` logs + an operator line). Ask the pusher to push it again under a free name: the old name + conflicts with the restored ref. The command writes to the ref store with the same compare-and-swap as a push, so the replicas can keep running. It changes the ref store only: it sends no push webhook and @@ -735,6 +737,7 @@ webhook events. | Postgres availability + advisory-lock waits at boot | The database is the single authority; boot hangs show in `pg_locks` | | `verify token: key store lookup failed` / `verify token: a registered key cannot be used` log lines | A git, LFS or REST request reached the key store or a registered key, and it failed. The line holds the cause (a cause of more than 300 runes loses its middle). A git or LFS client reads a text that does not hold the cause. For each kind, the git and LFS routes together write at most one line per minute in each `serve` process, the REST routes write at most one more, and a line says how many earlier lines of its kind were held back when some were | | `receive-pack : hook: … git cannot walk the history between two commits, so the update is refused` log line | A push named two commits whose history git cannot read, so the hook refused that update: as non-fast-forward under `no-force-push`, and with `is not a readable commit in this repository` under `verify-sig`. The line has the fields `git`, `old` and `new`. A pusher can cause it with a commit whose parent is missing. A `` that repeats it for ordinary pushes from several clients has a damaged history: check a fresh copy with `git fsck` | +| `receive-pack : hook: … a push deletes a ref that its replica does not list, and the ref store holds it, so the update is refused` log line | A client sent `git push :` for a ref that its replica's cache does not list, and the ref store holds the ref. The pusher read ` exists at in the ref store, but this replica does not list it`. The line has the fields `ref`, `sha` and `cause`. One line after a ref was created is normal: the client fetches and pushes again. A `` that repeats it has a replica whose cache stays behind: look for a `hydrate :` error on that replica | | `lane scan: requeued a lane whose work had no job` log line | A sync or push request lost its job and the [lane scan](#background-maintenance-in-a-fleet) enqueued it again. A `repo=` that repeats needs a look: its runs keep crashing | The opt-in [metrics listener](../reference/metrics.md) covers some of these signals; @@ -767,6 +770,7 @@ line can be up to 1 MiB. | Postgres availability and advisory-lock waits (What to monitor) | **log only** | Observe PostgreSQL itself (`pg_locks`) | | `verify token: …` lines (What to monitor) | **log only** | The two texts shown, at most one line for each kind per minute for the git and LFS routes together, and one more for the REST routes, in each `serve` process | | `git cannot walk the history between two commits` (What to monitor) | **log only** | A hook line, so metrics do not count it. Fields `git`, `old`, `new` and `cause` | +| `a push deletes a ref that its replica does not list` (What to monitor) | **log only** | A hook line, so metrics do not count it. Fields `ref`, `sha` and `cause` | | `lane scan: requeued a lane whose work had no job` (What to monitor) | **log only** | Fields `org`, `repo` and `lane` | | S3WAL `… conditional-write contention retries exhausted` (S3WAL signals) | `openstorage_s3wal_cas_contention_exhausted_total` by `scope` | Serve-process mutations only; push-side failures reach the client and hook stderr | | `WARN s3wal manifest approaching size cap` (S3WAL signals) | `openstorage_s3wal_manifest_size_guard_total{result="warn"}` | Counts when the once-per-repository warning is logged | diff --git a/internal/githttp/delete_absent_ref_e2e_test.go b/internal/githttp/delete_absent_ref_e2e_test.go new file mode 100644 index 00000000..f420c074 --- /dev/null +++ b/internal/githttp/delete_absent_ref_e2e_test.go @@ -0,0 +1,78 @@ +package githttp + +// git sends the old value of a ref that it deletes. When the replica's cache +// does not list the ref, git sends the zero id for it, so the command is +// zero to zero, and stock git's receive-pack treats it as a delete of a ref +// that does not exist: a no-op that it reports as ok, with a warning. The +// update hook gets the command as it came. A zero to zero swap in the ref +// store is an assertion, so the hook must answer the command itself, and the +// pusher must never read a Go trace. + +import ( + "io" + "os" + "strings" + "testing" + "time" + + "github.com/coder/openstorage/internal/ref" +) + +func TestDeleteOfAnAbsentRefIsAnsweredNotCrashed(t *testing.T) { + var serveLog syncBuffer + node, err := startBinaryNode(binPath, t.TempDir(), io.MultiWriter(os.Stderr, &serveLog)) + if err != nil { + t.Fatalf("start node: %v", err) + } + t.Cleanup(node.stop) + repo := registerRepo(t, "t-delete-absent") + work := initWorkRepo(t) + c1 := revParse(t, work, "HEAD") + token := mintTestToken(t, repo, "git:read,git:write", "") + mustGit(t, work, "push", remoteURL(node, repo, token), "HEAD:refs/heads/main") + noPush := mintTestToken(t, repo, "git:read,git:write", `[["refs/notes/*",["no-push"]],["*",[]]]`) + + const held = "refs/notes/held" + if won, err := testStore.CASRef(t.Context(), repo, held, ref.ZeroSHA, c1); err != nil || !won { + t.Fatalf("plant %s in the ref store: won=%v err=%v", held, won, err) + } + // The branch and tag names are the shapes of the delete in the runbook. + for _, tc := range []struct { + name, token, ref string + wantOK bool + wantText string + }{ + {"a tag that no store holds", token, "refs/tags/ghost", true, ""}, + {"a branch that no store holds", token, "refs/heads/ghost", true, ""}, + {"a notes ref that no store holds", token, "refs/notes/ghost", true, ""}, + {"a notes ref that the ref store holds", token, held, false, held + " exists at " + c1[:8]}, + {"a notes ref under a no-push policy", noPush, "refs/notes/ghost", false, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + logStart := len(serveLog.String()) + before := pgRefs(t, repo) + report, remote := rawReceivePack(t, node, repo, tc.token, false, nil, rawPushCommand{ref.ZeroSHA, ref.ZeroSHA, tc.ref}) + assertNoCrashText(t, remote) + if got := strings.HasPrefix(report[tc.ref], "ok"); got != tc.wantOK { + t.Fatalf("%s: report %q, want ok = %v\nremote:\n%s", tc.ref, report[tc.ref], tc.wantOK, remote) + } + if !strings.Contains(remote, tc.wantText) { + t.Fatalf("the pusher does not read %q:\n%s", tc.wantText, remote) + } + after := pgRefs(t, repo) + if len(after) != len(before) || after[held] != c1 || after[tc.ref] != before[tc.ref] { + t.Fatalf("refs changed: before %v, after %v", before, after) + } + if tc.wantText != "" && !tc.wantOK && tc.ref == held { + // serve logs the hook-errors file after the push ends. + pushLog := func() string { return serveLog.String()[logStart:] } + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !strings.Contains(pushLog(), held); { + time.Sleep(20 * time.Millisecond) + } + if !strings.Contains(pushLog(), held) { + t.Fatalf("serve's log does not name %s:\n%s", held, pushLog()) + } + } + }) + } +} diff --git a/internal/githttp/delete_absent_ref_test.go b/internal/githttp/delete_absent_ref_test.go new file mode 100644 index 00000000..b779227e --- /dev/null +++ b/internal/githttp/delete_absent_ref_test.go @@ -0,0 +1,182 @@ +package githttp + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/coder/openstorage/internal/ref" + "github.com/coder/openstorage/internal/ref/s3wal" +) + +// The update hook gets a command whose old and new values are both the zero +// id when a client deletes a ref that its replica does not list (see +// delete_absent_ref_e2e_test.go). A swap from nothing to nothing is an +// assertion in every ref store, so the hook must answer it before it swaps: +// ok when the store holds no such ref, a refusal when it holds one. +func TestUpdateHookAnswersAZeroToZeroCommand(t *testing.T) { + const held, ghost = "refs/notes/held", "refs/notes/ghost" + sha := strings.Repeat("c", 40) + dbURL := "sqlite://" + filepath.Join(t.TempDir(), "refs.sqlite") + st, err := ref.Open(t.Context(), dbURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if err := st.InitSchema(t.Context()); err != nil { + t.Fatal(err) + } + if won, err := st.CASRef(t.Context(), "t-refusal", held, ref.ZeroSHA, sha); err != nil || !won { + t.Fatalf("plant %s: won=%v err=%v", held, won, err) + } + records := hookEnv(t, dbURL) + update := func(name string) error { + return RunHook(t.Context(), []string{"update", name, ref.ZeroSHA, ref.ZeroSHA}) + } + + if err := update(ghost); err != nil { + t.Fatalf("a delete of a ref that the store does not hold = %v, want ok", err) + } + err = update(held) + if err == nil || !ShownToPusher(err) || !strings.Contains(err.Error(), held+" exists at "+sha[:8]) { + t.Fatalf("a delete of a ref that the store holds = %v, want a refusal that names the ref and its sha", err) + } + // The monitoring table promises the fields ref, sha and cause. + line := readRecords(t, records) + for _, want := range []string{"a push deletes a ref that its replica does not list", "ref=" + held, "sha=" + sha, "cause="} { + if !strings.Contains(line, want) { + t.Fatalf("the hook-errors file has no %q for %s:\n%s", want, held, line) + } + } + if got, err := st.GetRef(t.Context(), "t-refusal", held); err != nil || got != sha { + t.Fatalf("%s = %q, %v after the refused delete, want %s", held, got, err, sha) + } + if _, err := st.GetRef(t.Context(), "t-refusal", ghost); !errors.Is(err, ref.ErrNotFound) { + t.Fatalf("%s: %v, want ErrNotFound: the ok delete must not create it", ghost, err) + } +} + +// getRefFails is a ref store whose point read fails. +type getRefFails struct { + ref.RefStore + err error +} + +func (s getRefFails) GetRef(context.Context, string, string) (string, error) { return "", s.err } + +// The answer rests on a point read of the ref store (every store has one), so +// it holds on each store that the hook opens. PostgreSQL runs in the e2e lane. +// A failed read is a failure of the hook, never an ok and never a refusal that +// the pusher reads. +func TestAnswerDeleteOfAbsentRefOnEachStore(t *testing.T) { + const name = "refs/notes/x" + sha := strings.Repeat("d", 40) + sqlite, err := ref.Open(t.Context(), "sqlite://"+filepath.Join(t.TempDir(), "refs.sqlite")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { sqlite.Close() }) + if err := sqlite.InitSchema(t.Context()); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + store ref.RefStore + }{ + {"sqlite", sqlite}, + {"s3wal", ref.NewS3WAL(s3wal.NewMemCAS(), time.Now)}, + } { + t.Run(tc.name, func(t *testing.T) { + repo := "repo_" + tc.name + records, err := createHookErrors(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Setenv(EnvHookErrors, records) + if err := answerDeleteOfAbsentRef(t.Context(), tc.store, repo, name); err != nil { + t.Fatalf("absent: %v, want nil", err) + } + if won, err := tc.store.CASRef(t.Context(), repo, name, ref.ZeroSHA, sha); err != nil || !won { + t.Fatalf("create %s: won=%v err=%v", name, won, err) + } + if err := answerDeleteOfAbsentRef(t.Context(), tc.store, repo, name); err == nil || !ShownToPusher(err) { + t.Fatalf("held: %v, want a refusal that the pusher reads", err) + } + boom := errors.New("the ref store is down") + err = answerDeleteOfAbsentRef(t.Context(), getRefFails{tc.store, boom}, repo, name) + if !errors.Is(err, boom) || ShownToPusher(err) { + t.Fatalf("a failed read: %v, want the store's error, not shown to the pusher", err) + } + }) + } +} + +// recordsSwaps is a ref store that counts the swaps that the update hook makes. +type recordsSwaps struct { + ref.RefStore + swaps *[]string +} + +func (s recordsSwaps) CASRefGen(ctx context.Context, repo, name, old, new string) (bool, int64, error) { + *s.swaps = append(*s.swaps, name+" "+old[:8]+" "+new[:8]) + return s.RefStore.CASRefGen(ctx, repo, name, old, new) +} + +// Only a command with both values zero is answered without a swap. A create +// (old zero, new set) and a delete (old set, new zero) are swapped as before, +// whatever the store holds. +func TestUpdateHookSwapsEveryCommandThatIsNotZeroToZero(t *testing.T) { + const name = "refs/notes/swap" + sha := strings.Repeat("e", 40) + dbURL := "sqlite://" + filepath.Join(t.TempDir(), "refs.sqlite") + st, err := ref.Open(t.Context(), dbURL) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if err := st.InitSchema(t.Context()); err != nil { + t.Fatal(err) + } + hookEnv(t, dbURL) + var swaps []string + prev := WrapHookRefStore + t.Cleanup(func() { WrapHookRefStore = prev }) + WrapHookRefStore = func(_ context.Context, st ref.RefStore) (ref.RefStore, error) { + return recordsSwaps{st, &swaps}, nil + } + update := func(old, new string) error { + return RunHook(t.Context(), []string{"update", name, old, new}) + } + steps := []struct { + what, old, new string + wantSwap string + wantSHA string // the ref afterwards, "" for none + }{ + {"create", ref.ZeroSHA, sha, name + " 00000000 eeeeeeee", sha}, + {"zero to zero on a held ref", ref.ZeroSHA, ref.ZeroSHA, "", sha}, + {"delete", sha, ref.ZeroSHA, name + " eeeeeeee 00000000", ""}, + {"zero to zero on an absent ref", ref.ZeroSHA, ref.ZeroSHA, "", ""}, + } + for _, step := range steps { + swaps = nil + err := update(step.old, step.new) + if step.what == "zero to zero on a held ref" { + if err == nil { + t.Fatalf("%s: ok, want a refusal", step.what) + } + } else if err != nil { + t.Fatalf("%s: %v", step.what, err) + } + got := strings.Join(swaps, ",") + if got != step.wantSwap { + t.Fatalf("%s: swaps %q, want %q", step.what, got, step.wantSwap) + } + cur, err := st.GetRef(t.Context(), "t-refusal", name) + if step.wantSHA == "" && !errors.Is(err, ref.ErrNotFound) || step.wantSHA != "" && cur != step.wantSHA { + t.Fatalf("%s: ref %q, %v, want %q", step.what, cur, err, step.wantSHA) + } + } +} diff --git a/internal/githttp/hook.go b/internal/githttp/hook.go index 7ed8dd62..cd7de946 100644 --- a/internal/githttp/hook.go +++ b/internal/githttp/hook.go @@ -1862,6 +1862,9 @@ func updateHook(ctx context.Context, args []string) (retErr error) { return err } } + if old == ref.ZeroSHA && new == ref.ZeroSHA { + return answerDeleteOfAbsentRef(ctx, st, repo, refName) + } var inner ref.RefStore = st if testMode() { inner = injectCASDrop(inner) @@ -1901,6 +1904,30 @@ func updateHook(ctx context.Context, args []string) (retErr error) { return nil } +// answerDeleteOfAbsentRef answers a command whose old and new values are both +// the zero id. git sends that for `git push :` when the replica's +// cache does not list the ref, and a client can send it for any ref. The swap +// that the hook makes for a command is an assertion for it (a swap from nothing +// to nothing), and the command names no value to swap, so the hook decides +// from the ref store alone. A ref that the store does not hold makes it a +// delete of a ref that does not exist: stock git answers that with a warning +// and ok, and so does this. A ref that the store holds was created after the +// replica listed its refs, or the replica's cache lags: the pusher's expectation +// that the ref is absent is wrong, so the command is refused, as a lost swap +// is, and the operator gets a line. +func answerDeleteOfAbsentRef(ctx context.Context, st ref.RefStore, repo, refName string) error { + cur, err := st.GetRef(ctx, repo, refName) + switch { + case errors.Is(err, ref.ErrNotFound): + return nil + case err != nil: + return fmt.Errorf("update: read %s for a delete of a ref that the push named absent: %w", refName, err) + } + hookErrorsLogger().Warn("a push deletes a ref that its replica does not list, and the ref store holds it, so the update is refused", + "ref", refName, "sha", cur, "cause", "the replica's cache lags the ref store, or the ref was created since the push read the refs") + return refusef("update: %s exists at %.8s in the ref store, but this replica does not list it: fetch, then push the delete again", refName, cur) +} + // updateBarrierEachPath derives one ref's EnvUpdateBarrierEach handshake // path (hashed: ref names contain separators no flat handshake file can). func updateBarrierEachPath(prefix, refName string) string {