From 55ca354fdf357f4cd156a12fcb28b41035d3b013 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:05:21 -0400 Subject: [PATCH 1/5] perf(ci): run pr-verify's resolver before its trigger gate check-intent spent 472 GITHUB_TOKEN calls over the 2026-10-29 17:58-18:58Z window. The job annotations and post-verify-notes' verdict-or-none counts split them into 20 judgments (334 calls, 16.7 each) and about 24 runs that judged nothing (138 calls, 5.75 each). The skips sat in two clusters, 4-6 and 11-12 calls: the trigger gate's own reads, about 3 or 4, plus the resolver's 2 (no-single-pr, draft, author) or its pinned 8 (already-judged). The resolver only reads and already skips every run the brief names: a red run that only an approval does not lift, a PR whose head moved on, and a head already judged. So it now runs first, the gate runs only when the resolver proceeds, and the agree step only after the gate proceeds. A judged run makes the same reads as before. This keeps one run per head, not one per attempt, rather than skipping red runs, because PR authors and lanes rely on verify-on-red: - pr-verify.yml:3-6 and resolve-pr.mjs:5-9,588 (heldOnly) judge a red run held only by an approval-liftable hold or do-not-merge, so a person approves a judged head. - pr-feedback-dispatch.yml:18-35 turns a FAIL into threads that address-feedback works, or the pr-verify-failed label that ci-status holds. No state is carried. A per-head record in the Actions cache was rejected, because a restore extracts anywhere in the workspace (#9455, pr-require-checks.yml:5210-5213). A lookup-first precheck was rejected too, because it would add reads to all 20 judgments an hour. Co-Authored-By: Claude Opus 5.5 --- .../perf-github-token-cuts-30e1480a.md | 10 ++ .github/scripts/pr-verify/workflow.test.mjs | 18 +++ .github/workflows/pr-verify.yml | 130 ++++++++++-------- docs/conventions/pr-pipeline/README.md | 5 + 4 files changed, 107 insertions(+), 56 deletions(-) create mode 100644 .changes/pr-pipeline/perf-github-token-cuts-30e1480a.md diff --git a/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md b/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md new file mode 100644 index 00000000000..a028d80beda --- /dev/null +++ b/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md @@ -0,0 +1,10 @@ +--- +bump: patch +--- + +### Changed + +- pr-verify runs its read-only resolver before its trigger gate, so a run the resolver skips (a + head already judged or moved past, a draft, a run red for anything an approval does not lift, + an author it does not judge) no longer spends the gate's reads. A judged run makes the same + reads as before. diff --git a/.github/scripts/pr-verify/workflow.test.mjs b/.github/scripts/pr-verify/workflow.test.mjs index cc01e3c3bfb..9fbebfc0cdc 100644 --- a/.github/scripts/pr-verify/workflow.test.mjs +++ b/.github/scripts/pr-verify/workflow.test.mjs @@ -120,6 +120,24 @@ test("the upload is each job's only artifact step and follows the written verdic assert.doesNotMatch(text, /actions\/upload-artifact\/merge|gh api .*artifacts.* -X (POST|PUT)|actions\/download-artifact|actions\/cache/); }); +// The resolver reads only, and skips on its own reads a run whose head is +// judged, moved past, a draft, red for anything a person's approval does not +// lift, or not the lanes App's; so it runs before the trigger gate, whose +// reads then go only to a run the resolver would judge. The agree step runs +// only after the gate proceeds, so a gate refusal ends the run without a +// judgment and without a mismatch error. +test("the resolver runs before the trigger gate, and the agree step only after the gate proceeds", () => { + for (const job of JOBS) { + assert.ok(index("Resolve the pull request", job) < index("Check the trigger", job), job); + assert.ok(index("Check the trigger", job) < index("Assert the gate and the resolver agree", job), job); + assert.match(step("Resolve the pull request", job), /^ {8}if: steps\.kill\.outputs\.proceed == 'true'$/m, job); + assert.match(step("Check the trigger", job), /^ {8}if: steps\.pr\.outputs\.proceed == 'true'$/m, job); + assert.match(step("Assert the gate and the resolver agree", job), /^ {8}if: steps\.gate\.outputs\.proceed == 'true'$/m, job); + // Every judging step hangs off agree, and so does the missing-verdict failure. + assert.match(step("Fail without a verdict", job), /steps\.agree\.outcome == 'success'/, job); + } +}); + test("pr-merge.yml gives the gate an Actions token for the verdict reads only", () => { // babysit_verify_evidence.py reads BABYSIT_VERIFY_GH_TOKEN; the lane token // the gate runs gh with holds no Actions permission. diff --git a/.github/workflows/pr-verify.yml b/.github/workflows/pr-verify.yml index e0f47cdfd6f..21755598f77 100644 --- a/.github/workflows/pr-verify.yml +++ b/.github/workflows/pr-verify.yml @@ -116,25 +116,15 @@ jobs: echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable}); no verdict until the lost-run reconciler re-runs it" exit 1 - # The upstream run's actors, the re-runner and the PR author must all be - # on the trusted-actor list. - - name: Check the trigger - id: gate - if: steps.kill.outputs.proceed == 'true' - # Each API call keeps its rate-limit reading for the api-budget - # notice the job logs before its last step. - env: - API_BUDGET_TOKEN: github-token - uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] - with: - trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json - triggering-actor: ${{ github.triggering_actor }} - reconciler-workflow: .github/workflows/maintenance-reconcile-lost-runs.yml - ci-status-rerun-workflow: .github/workflows/pr-feedback-dispatch.yml - + # The resolver reads only, and skips on its own reads every run it + # would not judge (a head already judged or moved past, a draft, a run + # red for anything an approval does not lift, an author it does not + # judge), so it runs first and the gate's reads go only to a run it + # would judge. Nothing it writes reaches the model before the gate and + # the agree step pass. - name: Resolve the pull request id: pr - if: steps.gate.outputs.proceed == 'true' + if: steps.kill.outputs.proceed == 'true' env: GITHUB_TOKEN: ${{ github.token }} API_BUDGET_TOKEN: github-token @@ -152,11 +142,27 @@ jobs: export VERIFY_AUTHORS LANE_IDENTITY SPEC_FROM_BODY_APPS ISSUE_LANE_WEB_HOSTS node .base/.github/scripts/pr-verify/resolve-pr.mjs + # The upstream run's actors, the re-runner and the PR author must all be + # on the trusted-actor list. + - name: Check the trigger + id: gate + if: steps.pr.outputs.proceed == 'true' + # Each API call keeps its rate-limit reading for the api-budget + # notice the job logs before its last step. + env: + API_BUDGET_TOKEN: github-token + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + reconciler-workflow: .github/workflows/maintenance-reconcile-lost-runs.yml + ci-status-rerun-workflow: .github/workflows/pr-feedback-dispatch.yml + # The gate and the resolver read the PR separately; both must name the # same PR at the run's head, or nothing is judged. - name: Assert the gate and the resolver agree id: agree - if: steps.pr.outputs.proceed == 'true' + if: steps.gate.outputs.proceed == 'true' env: GATE_PR: ${{ steps.gate.outputs.pr-number }} GATE_HEAD: ${{ steps.gate.outputs.head-sha }} @@ -476,26 +482,15 @@ jobs: echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable}); no verdict until the lost-run reconciler re-runs it" exit 1 - # The upstream run's actors, the re-runner and the PR author must all be - # on the trusted-actor list. A re-run github-actions[bot] started passes - # only as the lost-run reconciler's recorded re-run of this run. - - name: Check the trigger - id: gate - if: steps.kill.outputs.proceed == 'true' - # Each API call keeps its rate-limit reading for the api-budget - # notice the job logs before its last step. - env: - API_BUDGET_TOKEN: github-token - uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] - with: - trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json - triggering-actor: ${{ github.triggering_actor }} - reconciler-workflow: .github/workflows/maintenance-reconcile-lost-runs.yml - ci-status-rerun-workflow: .github/workflows/pr-feedback-dispatch.yml - + # The resolver reads only, and skips on its own reads every run it + # would not judge (a head already judged or moved past, a draft, a run + # red for anything an approval does not lift, an author it does not + # judge), so it runs first and the gate's reads go only to a run it + # would judge. Nothing it writes reaches the model before the gate and + # the agree step pass. - name: Resolve the pull request id: pr - if: steps.gate.outputs.proceed == 'true' + if: steps.kill.outputs.proceed == 'true' env: GITHUB_TOKEN: ${{ github.token }} API_BUDGET_TOKEN: github-token @@ -513,11 +508,28 @@ jobs: export VERIFY_AUTHORS LANE_IDENTITY SPEC_FROM_BODY_APPS ISSUE_LANE_WEB_HOSTS node .base/.github/scripts/pr-verify/resolve-pr.mjs + # The upstream run's actors, the re-runner and the PR author must all be + # on the trusted-actor list. A re-run github-actions[bot] started passes + # only as the lost-run reconciler's recorded re-run of this run. + - name: Check the trigger + id: gate + if: steps.pr.outputs.proceed == 'true' + # Each API call keeps its rate-limit reading for the api-budget + # notice the job logs before its last step. + env: + API_BUDGET_TOKEN: github-token + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + reconciler-workflow: .github/workflows/maintenance-reconcile-lost-runs.yml + ci-status-rerun-workflow: .github/workflows/pr-feedback-dispatch.yml + # The gate and the resolver read the PR separately; both must name the # same PR at the run's head, or nothing is judged. - name: Assert the gate and the resolver agree id: agree - if: steps.pr.outputs.proceed == 'true' + if: steps.gate.outputs.proceed == 'true' env: GATE_PR: ${{ steps.gate.outputs.pr-number }} GATE_HEAD: ${{ steps.gate.outputs.head-sha }} @@ -839,26 +851,15 @@ jobs: echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable}); no verdict until the lost-run reconciler re-runs it" exit 1 - # The upstream run's actors, the re-runner and the PR author must all be - # on the trusted-actor list. A re-run github-actions[bot] started passes - # only as the lost-run reconciler's recorded re-run of this run. - - name: Check the trigger - id: gate - if: steps.kill.outputs.proceed == 'true' - # Each API call keeps its rate-limit reading for the api-budget - # notice the job logs before its last step. - env: - API_BUDGET_TOKEN: github-token - uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] - with: - trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json - triggering-actor: ${{ github.triggering_actor }} - reconciler-workflow: .github/workflows/maintenance-reconcile-lost-runs.yml - ci-status-rerun-workflow: .github/workflows/pr-feedback-dispatch.yml - + # The resolver reads only, and skips on its own reads every run it + # would not judge (a head already judged or moved past, a draft, a run + # red for anything an approval does not lift, an author it does not + # judge), so it runs first and the gate's reads go only to a run it + # would judge. Nothing it writes reaches the model before the gate and + # the agree step pass. - name: Resolve the pull request id: pr - if: steps.gate.outputs.proceed == 'true' + if: steps.kill.outputs.proceed == 'true' env: GITHUB_TOKEN: ${{ github.token }} API_BUDGET_TOKEN: github-token @@ -876,11 +877,28 @@ jobs: export VERIFY_AUTHORS LANE_IDENTITY SPEC_FROM_BODY_APPS ISSUE_LANE_WEB_HOSTS node .base/.github/scripts/pr-verify/resolve-pr.mjs + # The upstream run's actors, the re-runner and the PR author must all be + # on the trusted-actor list. A re-run github-actions[bot] started passes + # only as the lost-run reconciler's recorded re-run of this run. + - name: Check the trigger + id: gate + if: steps.pr.outputs.proceed == 'true' + # Each API call keeps its rate-limit reading for the api-budget + # notice the job logs before its last step. + env: + API_BUDGET_TOKEN: github-token + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + reconciler-workflow: .github/workflows/maintenance-reconcile-lost-runs.yml + ci-status-rerun-workflow: .github/workflows/pr-feedback-dispatch.yml + # The gate and the resolver read the PR separately; both must name the # same PR at the run's head, or nothing is judged. - name: Assert the gate and the resolver agree id: agree - if: steps.pr.outputs.proceed == 'true' + if: steps.gate.outputs.proceed == 'true' env: GATE_PR: ${{ steps.gate.outputs.pr-number }} GATE_HEAD: ${{ steps.gate.outputs.head-sha }} diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 841ed0d0b52..0b032a392aa 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -2146,6 +2146,11 @@ is re-run this way once (attempt 2 is never re-run; a later edit or label starts run), and a changed closing set or a removed spec comment leaves no time to compare, so those heads still wait for a person. +Each pr-verify job runs the resolver before the trigger gate. The resolver only reads, and it skips on +its own reads every run it would not judge: a head already judged or moved past, a draft, a run red +for anything an approval does not lift, an author it does not judge. So the gate's reads go only to +a run the resolver would judge, and nothing either writes reaches the model before both pass. + With `merge.require-verifier-verdict: true` the pr-merge lane reads the artifacts through their run records, since only steps of a run can upload its artifacts. A run is of verifier origin when it is `pr-verify.yml`, event `workflow_run`, on main, in this repository with a head on main. A From 408b51ec3e166ccc89e7b937eeca2ac41178ee81 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:49:36 -0400 Subject: [PATCH 2/5] ci: log GITHUB_TOKEN api-calls from intake-triage apply-verdict and pr-review-main record About 32% of the 2026-10-29 17:58-18:58Z window's GITHUB_TOKEN spend had no per-job api-calls notice. Two of the listed jobs are local and spend the job token without counting it: - intake-triage apply-verdict reads through resolve-config with the job token; that step now records under github-token and the job logs both lanes-app and github-token, as work-unblock's label-unblocked does. - pr-review-main record POSTs one check run per verdict file with no checkout of log-api-budget; it counts its POSTs and ends with the notice in api_budget_emit_calls' shape, pinned against the emitter by a test. Co-Authored-By: Claude Opus 5.5 --- .../log-api-budget/log-api-budget.test.mjs | 25 ++++++++++++++++++- .github/workflows/intake-triage.yml | 6 ++++- .github/workflows/pr-review-main.yml | 13 ++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/actions/log-api-budget/log-api-budget.test.mjs b/.github/actions/log-api-budget/log-api-budget.test.mjs index 0eab2dd69fb..f5420e6a68c 100644 --- a/.github/actions/log-api-budget/log-api-budget.test.mjs +++ b/.github/actions/log-api-budget/log-api-budget.test.mjs @@ -785,7 +785,8 @@ test("the action writes the content-creation notice after the token lines", () = // content-creating writes. test("each heavy lane job names its API_BUDGET_TOKEN and ends with the log-api-budget step", () => { const jobs = { - "intake-triage.yml": { "born-triaged": "github-token", "apply-verdict": "lanes-app", "refine-select": "github-token", "apply-refinement": "lanes-app" }, + // apply-verdict's resolve-config reads with the job token, recorded under github-token. + "intake-triage.yml": { "born-triaged": "github-token", "apply-verdict": ["lanes-app", "lanes-app github-token"], "refine-select": "github-token", "apply-refinement": "lanes-app" }, "work-resolve-issue.yml": { admit: "github-token", "pick-slot": "github-token", "report-statuses": "github-token" }, "pr-verify.yml": { "check-intent": "github-token", "check-intent-subscription": "github-token", "check-intent-fallback": "github-token" }, // label-unblocked's ACTIONS_TOKEN client records under github-token, so @@ -814,6 +815,28 @@ test("each heavy lane job names its API_BUDGET_TOKEN and ends with the log-api-b } }); +// pr-review-main's record job has no checkout of the action, and its only +// calls are one check-run POST per verdict file, so it counts them itself and +// ends with the notice in api_budget_emit_calls' shape. +test("pr-review-main's record job ends with its own api-calls notice in the emitter's shape", () => + withDir((dir) => { + const text = readFileSync(new URL("../../workflows/pr-review-main.yml", import.meta.url), "utf8"); + const block = /\n record:\n([\s\S]*?)(?=\n [a-z0-9-]+:\n|$)/.exec(text)?.[1] ?? ""; + const last = block.split(/\n {6}- name: /).at(-1); + assert.match(last, /^Log the API calls at the end\n {8}if: always\(\)\n/); + const run = /\n {8}run: \|\n((?: {10}[^\n]*\n?)+)/.exec(last)?.[1].replace(/^ {10}/gm, ""); + assert.ok(run, "the step has a run block"); + const lib = new URL("api-budget.bash", import.meta.url).pathname; + const emitted = (count) => { + if (count !== undefined) writeFileSync(join(dir, "pr-review-main-calls"), `${count}\n`); + return execFileSync("bash", ["-c", run], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir } }); + }; + const shape = execFileSync("bash", ["-c", `. "${lib}"; api_budget_emit_calls github-token`], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir } }); + assert.equal(emitted(), shape, "no count file is 0, as the emitter writes with no reading"); + assert.equal(emitted(2), shape.replace("github-token=0", "github-token=2")); + assert.equal(emitted("2; rm"), shape, "anything but a count is 0"); + })); + // Jobs that hold the job token but call GitHub only through actions the // counter cannot see still end by logging it, so each run carries its // api-calls line, 0 when no step of the job's own called GitHub. diff --git a/.github/workflows/intake-triage.yml b/.github/workflows/intake-triage.yml index c7f2dcd865a..1b8a6c10877 100644 --- a/.github/workflows/intake-triage.yml +++ b/.github/workflows/intake-triage.yml @@ -459,9 +459,13 @@ jobs: TRUSTED_ACTORS_PATH: .base/.github/standards/trusted-actors/trusted-actors.json run: node .base/.github/scripts/intake-triage/apply-verdict.mjs --check + # Its reads go through the job token, counted under github-token for + # the job-end api-calls notice. - name: Resolve the config id: config if: steps.who.outputs.proceed == 'true' + env: + API_BUDGET_TOKEN: github-token uses: ./.base/.github/actions/resolve-config # zizmor: ignore[self-repository] with: lane: intake-triage @@ -536,7 +540,7 @@ jobs: timeout-minutes: 1 uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository] with: - tokens: lanes-app + tokens: lanes-app github-token # Refines a thin issue (#9215): one whose verdict flags intent (no exact # outcome) or whose body has no task list (no checkable criteria). It also diff --git a/.github/workflows/pr-review-main.yml b/.github/workflows/pr-review-main.yml index 4c9e9a509c0..d48884e37c5 100644 --- a/.github/workflows/pr-review-main.yml +++ b/.github/workflows/pr-review-main.yml @@ -1472,6 +1472,8 @@ jobs: VERDICTS: ${{ runner.temp }}/pr-review-main-verdicts run: | shopt -s nullglob + calls=0 + trap 'echo "$calls" >"$RUNNER_TEMP/pr-review-main-calls"' EXIT for file in "$VERDICTS"/*/verdict.json; do lane=$(jq -r '.lane' "$file"); head=$(jq -r '.head_sha' "$file") critical=$(jq -r '.critical' "$file"); findings=$(jq -r '.findings' "$file") @@ -1480,6 +1482,7 @@ jobs: # Neutral either way: the gate reads the title, and a red check would # send the fix-ci lane after a review finding. conclusion=neutral + calls=$((calls + 1)) gh api -X POST "repos/$GITHUB_REPOSITORY/check-runs" \ -f name="pr-review-main: $lane" -f head_sha="$head" -f status=completed \ -f conclusion="$conclusion" \ @@ -1487,3 +1490,13 @@ jobs: -f "output[summary]=pr-review-main reviewed $head from main: $findings finding(s), $critical CRITICAL. The run's summary lists them." >/dev/null echo "recorded $lane $head findings=$findings critical=$critical" done + + # The hub's api-calls notice, in log-api-budget's api_budget_emit_calls + # shape: this job has no checkout of that action, and its only calls + # are the check-run POSTs above, counted as they are sent. + - name: Log the API calls at the end + if: always() + run: | + calls=$(cat "$RUNNER_TEMP/pr-review-main-calls" 2>/dev/null) + [[ $calls =~ ^[0-9]{1,6}$ ]] || calls=0 + echo "::notice title=api-calls::github-token=$calls not-modified=0 graphql=0" From 0b7c6e0eb3f9c2dda31f72ffc57c61e38efad699 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:09:00 -0400 Subject: [PATCH 3/5] feat(ci): pace timer-driven lanes on the GITHUB_TOKEN burn rate A budget trip (the breaker share of the header's limit left) turns the kill switch on and stops every lane. In the 2026-10-29 17:58-18:58Z window the pool reached used=4289 remaining=711 about 4 minutes before reset. The scheduled lanes were still spending at full rate then: reconcile at 68.5 calls per run, pr-unblock-dispatch at 20.6. On a schedule tick, pace.mjs reads the pool once and projects the remaining at the reset from the window's burn rate so far. When that falls under limits.api github-token `pace`, an even-numbered tick writes paced=true. `pace` is a new config value, default 0.2 in rate-limits.json, validated like the breaker and allowed in the schema. The lost-run reconciler and pr-unblock-dispatch skip their work on a paced tick and end green. Odd ticks still run, so a paced lane runs at half its cadence rather than stopping, and the reconciler keeps re-running the runs a trip stops. Any other event, or a failed read, runs as before. The breaker stays the backstop. Co-Authored-By: Claude Opus 5.5 --- .../perf-github-token-cuts-30e1480a.md | 10 +- .github/actions/log-api-budget/pace.mjs | 76 +++++++++ .github/actions/log-api-budget/pace.test.mjs | 144 ++++++++++++++++++ .github/actions/log-api-budget/rate-guard.mjs | 3 +- .../actions/log-api-budget/rate-limits.json | 2 +- .../pr-unblock/workflow-gates.test.mjs | 6 +- .../maintenance-reconcile-lost-runs.yml | 17 +++ .github/workflows/pr-unblock-dispatch.yml | 19 +++ docs/conventions/pr-pipeline.yaml | 5 +- docs/conventions/pr-pipeline/README.md | 2 +- .../pr-pipeline/pr-pipeline.schema.json | 6 + 11 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 .github/actions/log-api-budget/pace.mjs create mode 100644 .github/actions/log-api-budget/pace.test.mjs diff --git a/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md b/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md index a028d80beda..e3de6e9bd32 100644 --- a/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md +++ b/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md @@ -1,7 +1,15 @@ --- -bump: patch +bump: minor --- +### Added + +- `limits.api` github-token `pace` (default 0.2, in rate-limits.json): the lost-run reconciler + and pr-unblock-dispatch skip every other scheduled tick (the even run numbers) while the + GITHUB_TOKEN pool's remaining at its reset, projected from the window's burn rate so far, is + under that share of the header's limit, so the timer lanes slow to half cadence before the + breaker stops every lane. Any other event, or a failed read, runs as before. + ### Changed - pr-verify runs its read-only resolver before its trigger gate, so a run the resolver skips (a diff --git a/.github/actions/log-api-budget/pace.mjs b/.github/actions/log-api-budget/pace.mjs new file mode 100644 index 00000000000..56682f98fe0 --- /dev/null +++ b/.github/actions/log-api-budget/pace.mjs @@ -0,0 +1,76 @@ +// The burn-rate pacer for timer-driven lanes. A budget trip (the breaker +// share of the header's limit left, rate-limits.json) turns the kill switch +// on and stops every lane; this slows the timer-driven ones first. On a +// schedule tick it reads the GITHUB_TOKEN pool once, projects the remaining +// at the window's reset from the burn rate the window has shown so far, and +// writes paced=true when that falls under `pace` of the header's limit +// (limits.api github-token pace, default in rate-limits.json). Only an +// even-numbered tick is paced, so a paced lane runs at half its cadence +// rather than stopping: the lost-run reconciler, which re-runs the runs a +// breaker trip stops, keeps working while the pool recovers. A paced tick +// skips its work and ends green. Any other event, a reading that cannot +// project, a run number that is no number, or a failed read is +// paced=false: the breaker stays the backstop. +// +// node pace.mjs EVENT_NAME, RUN_NUMBER, GITHUB_TOKEN, GITHUB_REPOSITORY +// and GITHUB_OUTPUT; RESOLVED_CONFIG optional + +import { appendFileSync } from "node:fs"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; + +import { apiLimits, countRequests } from "./rate-guard.mjs"; + +// GitHub's primary rate-limit window. +const WINDOW_SECONDS = 3600; +// Before this much of the window has passed, the burn rate is too short to project. +const MIN_ELAPSED_SECONDS = 60; + +const round = (value) => Math.round(value * 100) / 100; + +// { paced, projected } for one reading at `now` (epoch seconds): projected is +// the share of the header's limit left at the reset, at the window's burn +// rate so far, clamped to 0..1. +export function paceDecision({ limit, remaining, reset } = {}, share, now) { + const left = reset - now; + if (![limit, remaining, reset].every(Number.isFinite) || limit <= 0 || remaining < 0 || remaining > limit || left < 0 || !(share > 0)) { + return { paced: false, projected: null }; + } + const elapsed = WINDOW_SECONDS - Math.min(left, WINDOW_SECONDS); + const used = limit - remaining; + const atReset = elapsed < MIN_ELAPSED_SECONDS ? remaining : remaining - (used * left) / elapsed; + const projected = Math.max(0, atReset) / limit; + return { paced: projected < share, projected: round(projected) }; +} + +export async function main(env = process.env, { fetchImpl = fetch, now = () => Date.now() / 1000 } = {}) { + let paced = false; + if (env.EVENT_NAME === "schedule") { + try { + const share = apiLimits({ resolvedPath: env.RESOLVED_CONFIG })["github-token"].pace; + // Counted for the job-end api-calls notice like any guarded call. + const response = await countRequests(fetchImpl)(`${env.GITHUB_API_URL ?? "https://api.github.com"}/repos/${env.GITHUB_REPOSITORY}`, { + headers: { Accept: "application/vnd.github+json", Authorization: `Bearer ${env.GITHUB_TOKEN}`, "X-GitHub-Api-Version": "2022-11-28" }, + }); + const header = (name) => (response.headers.has(name) ? Number(response.headers.get(name)) : Number.NaN); + const reading = { limit: header("x-ratelimit-limit"), remaining: header("x-ratelimit-remaining"), reset: header("x-ratelimit-reset") }; + const decision = response.ok ? paceDecision(reading, share, now()) : { paced: false, projected: null }; + const even = /^[0-9]+$/.test(env.RUN_NUMBER ?? "") && Number(env.RUN_NUMBER) % 2 === 0; + paced = decision.paced && even; + const why = paced ? "; this tick skips its work" : decision.paced ? "; an odd tick runs at half cadence" : ""; + if (decision.projected !== null) { + console.log(`::notice title=api-pace::github-token projected=${decision.projected} pace=${share} paced=${paced}${why}`); + } + } catch (error) { + console.log(`::warning::the pacer could not read the GITHUB_TOKEN pool (${error.message}); the tick runs`); + paced = false; + } + } + appendFileSync(env.GITHUB_OUTPUT, `paced=${paced}\n`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.log(`::warning::the pacer failed (${error.message}); the tick runs`); + }); +} diff --git a/.github/actions/log-api-budget/pace.test.mjs b/.github/actions/log-api-budget/pace.test.mjs new file mode 100644 index 00000000000..5e96f1e5b4f --- /dev/null +++ b/.github/actions/log-api-budget/pace.test.mjs @@ -0,0 +1,144 @@ +// The burn-rate pacer's rules (pace.mjs): a timer-driven run is paced when +// the GITHUB_TOKEN pool's remaining at its reset, projected at the burn rate +// the window has shown so far, falls under `pace` of the header's limit, the +// share the breaker is also stated in (pr-pipeline.schema.json api-pool). +// Expected values are worked by hand from limit, remaining and reset. +import assert from "node:assert/strict"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { main, paceDecision } from "./pace.mjs"; +import { apiLimits } from "./rate-guard.mjs"; +import { tempDir } from "../../../lib/test-tmp.mjs"; + +const NOW = 1_800_000_000; + +test("the pace share is a config value: rate-limits.json's github-token default, under limits.api", () => { + assert.equal(apiLimits({ resolvedPath: "" })["github-token"].pace, 0.2); + const dir = tempDir("pace-config-"); + const resolved = join(dir, "resolved.json"); + writeFileSync(resolved, JSON.stringify({ limits: { api: { "github-token": { pace: 0.35 } } } })); + assert.equal(apiLimits({ resolvedPath: resolved })["github-token"].pace, 0.35); + // The schema lets limits.api set it, in the same range as the breaker. + const schema = JSON.parse(readFileSync(new URL("../../../docs/conventions/pr-pipeline/pr-pipeline.schema.json", import.meta.url), "utf8")); + assert.deepEqual( + { ...schema.$defs["api-pool"].properties.pace, description: undefined }, + { ...schema.$defs["api-pool"].properties.breaker, description: undefined }, + ); +}); + +test("a run is paced when the projected remaining at reset falls under the share", () => { + // 30 minutes in: 2,000 of 5,000 used, so 2,000 more by the reset leaves 1,000, 20%. + const half = { limit: 5000, remaining: 3000, reset: NOW + 1800 }; + assert.deepEqual(paceDecision(half, 0.2, NOW), { paced: false, projected: 0.2 }); + // 2,100 used: 2,100 more leaves 800, 16%. + assert.deepEqual(paceDecision({ ...half, remaining: 2900 }, 0.2, NOW), { paced: true, projected: 0.16 }); + // The evidence window's reading: used 4,289, remaining 711, about 4 minutes to reset. + assert.equal(paceDecision({ limit: 5000, remaining: 711, reset: NOW + 252 }, 0.2, NOW).paced, true); + // A slow burn early in the window: 100 used in 10 minutes projects 500 more, 4,400 left. + assert.deepEqual(paceDecision({ limit: 5000, remaining: 4900, reset: NOW + 3000 }, 0.2, NOW), { paced: false, projected: 0.88 }); + // A projection under zero is clamped to it. + assert.deepEqual(paceDecision({ limit: 1000, remaining: 100, reset: NOW + 1800 }, 0.2, NOW), { paced: true, projected: 0 }); +}); + +test("the first minute projects nothing, so only the remaining itself can pace it", () => { + assert.deepEqual(paceDecision({ limit: 5000, remaining: 4000, reset: NOW + 3590 }, 0.2, NOW), { paced: false, projected: 0.8 }); + assert.deepEqual(paceDecision({ limit: 5000, remaining: 900, reset: NOW + 3590 }, 0.2, NOW), { paced: true, projected: 0.18 }); +}); + +test("a reading that cannot project, or a share of 0, never paces", () => { + for (const reading of [{}, { limit: 0, remaining: 0, reset: NOW + 60 }, { limit: 5000, remaining: "na", reset: NOW + 60 }, { limit: 5000, remaining: 100, reset: NOW - 10 }, { limit: 5000, remaining: 6000, reset: NOW + 60 }]) { + assert.equal(paceDecision(reading, 0.2, NOW).paced, false, JSON.stringify(reading)); + } + assert.equal(paceDecision({ limit: 5000, remaining: 10, reset: NOW + 1800 }, 0, NOW).paced, false); +}); + +// main's one read: a 200 answer's x-ratelimit headers. +const answer = (headers, status = 200) => async () => new Response("{}", { status, headers }); +const run = async (env, fetchImpl) => { + const dir = tempDir("pace-"); + const output = join(dir, "output"); + writeFileSync(output, ""); + const lines = []; + const log = console.log; + console.log = (line) => lines.push(line); + try { + await main({ GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GITHUB_OUTPUT: output, RUNNER_TEMP: dir, EVENT_NAME: "schedule", RUN_NUMBER: "42", ...env }, { fetchImpl, now: () => NOW }); + } finally { + console.log = log; + } + return { output: readFileSync(output, "utf8"), lines }; +}; + +test("on a schedule tick main reads the pool once and writes paced, with a notice", async () => { + let calls = 0; + const fetchImpl = async (...args) => { + calls += 1; + return answer({ "x-ratelimit-limit": "5000", "x-ratelimit-remaining": "2900", "x-ratelimit-used": "2100", "x-ratelimit-reset": String(NOW + 1800), "x-ratelimit-resource": "core" })(...args); + }; + const got = await run({}, fetchImpl); + assert.equal(calls, 1); + assert.equal(got.output, "paced=true\n"); + assert.deepEqual(got.lines, ["::notice title=api-pace::github-token projected=0.16 pace=0.2 paced=true; this tick skips its work"]); +}); + +// Pacing halves a lane's cadence rather than stopping it: a paced +// projection still runs the odd-numbered ticks, so the lost-run reconciler +// keeps re-running lost runs while the pool recovers. +test("a paced projection skips only the even-numbered ticks", async () => { + const low = answer({ "x-ratelimit-limit": "5000", "x-ratelimit-remaining": "2900", "x-ratelimit-reset": String(NOW + 1800) }); + assert.equal((await run({ RUN_NUMBER: "42" }, low)).output, "paced=true\n"); + const odd = await run({ RUN_NUMBER: "43" }, low); + assert.equal(odd.output, "paced=false\n"); + assert.deepEqual(odd.lines, ["::notice title=api-pace::github-token projected=0.16 pace=0.2 paced=false; an odd tick runs at half cadence"]); + // A run number that is no number runs the tick. + for (const number of ["", "4x", "-2"]) { + assert.equal((await run({ RUN_NUMBER: number }, low)).output, "paced=false\n", JSON.stringify(number)); + } +}); + +test("any other event, an unreadable answer or a failed read is never paced", async () => { + let calls = 0; + const counting = async () => { + calls += 1; + return new Response("{}", { status: 200 }); + }; + assert.equal((await run({ EVENT_NAME: "workflow_dispatch" }, counting)).output, "paced=false\n"); + assert.equal((await run({ EVENT_NAME: "push" }, counting)).output, "paced=false\n"); + assert.equal(calls, 0, "no read off a schedule tick"); + assert.equal((await run({}, counting)).output, "paced=false\n", "no headers"); + assert.equal((await run({}, answer({}, 502))).output, "paced=false\n"); + const throws = async () => { + throw new Error("down"); + }; + const failed = await run({}, throws); + assert.equal(failed.output, "paced=false\n"); + assert.match(failed.lines.join("\n"), /::warning::/); +}); + +// The timer-driven lanes that read the pacer: each reads it once on the +// event, never fails on it, and gates every step that spends the pool. +test("each paced timer lane reads the pacer before its work and gates every spending step on it", () => { + const lanes = { + "maintenance-reconcile-lost-runs.yml": { job: "reconcile", gated: ["Find lost runs", "Re-run lost runs"] }, + "pr-unblock-dispatch.yml": { job: "dispatch", gated: ["Install resolve-config's dependencies", "Restore the dispatch record", "Dispatch pr-unblock for conflicting, review-tier Dependabot, failing lanes App and Markdown-changing PRs"] }, + }; + for (const [file, { job, gated }] of Object.entries(lanes)) { + const text = readFileSync(new URL(`../../workflows/${file}`, import.meta.url), "utf8"); + const block = new RegExp(`\\n ${job}:\\n([\\s\\S]*?)(?=\\n [a-z0-9-]+:\\n|$)`).exec(text)?.[1] ?? ""; + // Less the blank and comment lines that lead into the next step. + const steps = block.split(/\n(?= {6}- name: )/).slice(1).map((s) => ({ name: /^ {6}- name: (.*)$/m.exec(s)[1], s: s.replace(/(\n *(#[^\n]*)?)+$/, "") })); + const at = (name) => steps.findIndex((step) => step.name === name); + const pace = steps[at("Pace the tick on the GITHUB_TOKEN burn rate")]?.s ?? ""; + assert.match(pace, /\n {8}id: pace\n/, file); + assert.match(pace, /\n {8}continue-on-error: true\n/, file); + assert.match(pace, /\n {10}EVENT_NAME: \$\{\{ github\.event_name \}\}\n/, file); + assert.match(pace, /\n {10}RUN_NUMBER: \$\{\{ github\.run_number \}\}\n/, file); + assert.match(pace, /\n {8}run: node \.base\/\.github\/actions\/log-api-budget\/pace\.mjs\n?$/, file); + for (const name of gated) { + assert.ok(at(name) > at("Pace the tick on the GITHUB_TOKEN burn rate"), `${file}: ${name} follows the pacer`); + assert.match(steps[at(name)].s, /steps\.pace\.outputs\.paced != 'true'/, `${file}: ${name}`); + } + } +}); diff --git a/.github/actions/log-api-budget/rate-guard.mjs b/.github/actions/log-api-budget/rate-guard.mjs index 7c61d56199d..430116729e7 100644 --- a/.github/actions/log-api-budget/rate-guard.mjs +++ b/.github/actions/log-api-budget/rate-guard.mjs @@ -53,7 +53,8 @@ const DEFAULTS = new URL("./rate-limits.json", import.meta.url); // belong to content-creation alone. const SHARE = (value) => typeof value === "number" && value >= 0 && value <= 1; const COUNT = (value) => Number.isInteger(value) && value >= 1; -const FIELDS = { plan: (value) => SHARE(value) && value > 0, "writer-pause": SHARE, breaker: SHARE, "read-floor": SHARE, "limit-cap": COUNT }; +// pace, optional, is pace.mjs's share for timer-driven lanes. +const FIELDS = { plan: (value) => SHARE(value) && value > 0, "writer-pause": SHARE, breaker: SHARE, "read-floor": SHARE, "limit-cap": COUNT, pace: SHARE }; const CONTENT_FIELDS = { ...FIELDS, "per-minute": COUNT, "per-hour": COUNT }; const REQUIRED = ["plan", "writer-pause", "breaker", "read-floor"]; diff --git a/.github/actions/log-api-budget/rate-limits.json b/.github/actions/log-api-budget/rate-limits.json index ac6652ddd6f..af023512a2d 100644 --- a/.github/actions/log-api-budget/rate-limits.json +++ b/.github/actions/log-api-budget/rate-limits.json @@ -1,5 +1,5 @@ { - "github-token": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1, "limit-cap": 1000 }, + "github-token": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1, "limit-cap": 1000, "pace": 0.2 }, "lanes-app": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1 }, "graphql": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1 }, "owner-token": { "plan": 0.7, "writer-pause": 0.4, "breaker": 0.2, "read-floor": 0.1 }, diff --git a/.github/scripts/pr-unblock/workflow-gates.test.mjs b/.github/scripts/pr-unblock/workflow-gates.test.mjs index 5bc8790bf79..16f7db839c0 100644 --- a/.github/scripts/pr-unblock/workflow-gates.test.mjs +++ b/.github/scripts/pr-unblock/workflow-gates.test.mjs @@ -745,15 +745,19 @@ test("pr-unblock-dispatch.yml gates a hand run and every re-run, and no first pu assert.equal(runs("schedule", "2"), true); assert.equal(runs("schedule", "1", "false"), false); // A skipped gate lets the dispatch step run; a checked one must proceed. - const dispatches = (outcome, proceed = "") => + const dispatches = (outcome, proceed = "", paced = "false") => evaluate(stepIf(DISPATCHER, "Dispatch pr-unblock for conflicting, review-tier Dependabot, failing lanes App and Markdown-changing PRs"), { "steps.kill.outputs.proceed": "true", "steps.gate.outcome": outcome, "steps.gate.outputs.proceed": proceed, + "steps.pace.outputs.paced": paced, }); assert.equal(dispatches("skipped"), true); assert.equal(dispatches("success", "true"), true); assert.equal(dispatches("success", "false"), false); + // A tick the burn-rate pacer paces dispatches nothing (log-api-budget/pace.mjs). + assert.equal(dispatches("skipped", "", "true"), false); + assert.equal(dispatches("skipped", "", ""), true, "a pacer that wrote nothing"); }); test("pr-unblock-dispatch.yml ticks five minutes after each lost-run reconciler tick", () => { diff --git a/.github/workflows/maintenance-reconcile-lost-runs.yml b/.github/workflows/maintenance-reconcile-lost-runs.yml index 067f871b629..158eb81498f 100644 --- a/.github/workflows/maintenance-reconcile-lost-runs.yml +++ b/.github/workflows/maintenance-reconcile-lost-runs.yml @@ -124,8 +124,24 @@ jobs: value: ${{ vars.CLAUDE_LANES_DISABLED }} repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + # A schedule tick skips its work while the GITHUB_TOKEN pool's remaining + # at reset, projected at this window's burn rate, is under limits.api + # github-token pace (rate-limits.json), on even run numbers only, so the + # timer lanes slow to half cadence before the breaker stops every lane. + # Any other event, or a read that fails, runs as before + # (.github/actions/log-api-budget/pace.mjs). + - name: Pace the tick on the GITHUB_TOKEN burn rate + id: pace + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + RUN_NUMBER: ${{ github.run_number }} + run: node .base/.github/actions/log-api-budget/pace.mjs + - name: Find lost runs id: plan + if: steps.pace.outputs.paced != 'true' env: MODE: plan PLAN_PATH: ${{ runner.temp }}/reconcile-plan.json @@ -174,6 +190,7 @@ jobs: retention-days: 1 - name: Re-run lost runs + if: steps.pace.outputs.paced != 'true' env: MODE: act PLAN_PATH: ${{ runner.temp }}/reconcile-plan.json diff --git a/.github/workflows/pr-unblock-dispatch.yml b/.github/workflows/pr-unblock-dispatch.yml index 56bf39fb2ab..5101d81ef1f 100644 --- a/.github/workflows/pr-unblock-dispatch.yml +++ b/.github/workflows/pr-unblock-dispatch.yml @@ -138,12 +138,29 @@ jobs: echo "::error::${REASON:-untrusted-actor}: this run was not started by a listed trusted human" exit 1 + # A schedule tick skips its work while the GITHUB_TOKEN pool's remaining + # at reset, projected at this window's burn rate, is under limits.api + # github-token pace (rate-limits.json), on even run numbers only, so the + # timer lanes slow to half cadence before the breaker stops every lane. + # Any other event, or a read that fails, runs as before + # (.github/actions/log-api-budget/pace.mjs). + - name: Pace the tick on the GITHUB_TOKEN burn rate + id: pace + if: steps.kill.outputs.proceed == 'true' + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + RUN_NUMBER: ${{ github.run_number }} + run: node .base/.github/actions/log-api-budget/pace.mjs + # dispatch.mjs reads the base config's `limits` through the base's # resolve-config, which needs its dependencies. - name: Install resolve-config's dependencies if: >- steps.kill.outputs.proceed == 'true' && (steps.gate.outcome == 'skipped' || steps.gate.outputs.proceed == 'true') + && steps.pace.outputs.paced != 'true' run: npm ci --ignore-scripts --omit=dev --prefix .base/.github/actions/resolve-config # The newest record a run of this workflow saved; none on the first run. @@ -151,6 +168,7 @@ jobs: if: >- steps.kill.outputs.proceed == 'true' && (steps.gate.outcome == 'skipped' || steps.gate.outputs.proceed == 'true') + && steps.pace.outputs.paced != 'true' continue-on-error: true uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -163,6 +181,7 @@ jobs: if: >- steps.kill.outputs.proceed == 'true' && (steps.gate.outcome == 'skipped' || steps.gate.outputs.proceed == 'true') + && steps.pace.outputs.paced != 'true' env: GITHUB_TOKEN: ${{ github.token }} API_BUDGET_TOKEN: github-token diff --git a/docs/conventions/pr-pipeline.yaml b/docs/conventions/pr-pipeline.yaml index 428752f60a0..b91c3092c9e 100644 --- a/docs/conventions/pr-pipeline.yaml +++ b/docs/conventions/pr-pipeline.yaml @@ -832,8 +832,9 @@ limits: # GitHub's documented GITHUB_TOKEN limit); plan and read-floor are shares of # it, writer-pause and breaker shares of the header's limit; content-creation # adds per-minute and per-hour. The rate guard reads only read-floor and - # limit-cap, to warn; the others are thresholds meant for - # the hub's limits sampler, which nothing here enforces. Their + # limit-cap, to warn; github-token's pace (projected share left at reset) + # paces timer-driven lanes (log-api-budget/pace.mjs); the others are + # thresholds meant for the hub's limits sampler, which nothing here enforces. Their # defaults live in .github/actions/log-api-budget/rate-limits.json, held # for a person like this file; add an `api:` entry here only to override one. diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 0b032a392aa..bec900e3696 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -2832,7 +2832,7 @@ with its default. | `fix-docs-per-run` | 2 | fix-docs dispatches one pr-unblock-dispatch run sends; at most 20. | | `admission-factor` | 0.8 | Share of the lane's hourly merge rate admitted per hour for the queued cap ([WIP cap](#wip-cap)); greater than 0, at most 1. | | `admission-min-per-hour` | 1 | Admissions per hour the queued cap never falls below, so a day with no lane merges cannot hold admissions shut; 0 to 100, and 0 removes the floor. | -| `api` | schema | GitHub API margins per pool (`github-token`, `lanes-app`, `graphql`, `owner-token`, `content-creation`). `limit-cap` sets the pool's planned budget: planned = min(header limit, `limit-cap`), used = header limit - header remaining, left = planned - used. `github-token` defaults it to GitHub's documented GITHUB_TOKEN limit, so a higher header limit is headroom, never a target; the other pools have no cap and plan to their header limit. `plan` and `read-floor` are shares of the planned budget; `writer-pause` and `breaker` are shares of the header's limit, by its remaining; `content-creation` adds `per-minute` and `per-hour`. Their defaults are in `.github/actions/log-api-budget/rate-limits.json`, the only copy, which every merge hold list floors, so a change to a default waits for a person. The rate guard (`.github/actions/log-api-budget/rate-guard.mjs`, `planOf`) reads only `read-floor` and `limit-cap`, to warn once while calls continue, and stops only on the header's remaining and GitHub's refusals; the others are thresholds meant for the hub's limits sampler, and nothing in this repository enforces them. resolve-config passes the block through as set. | +| `api` | schema | GitHub API margins per pool (`github-token`, `lanes-app`, `graphql`, `owner-token`, `content-creation`). `limit-cap` sets the pool's planned budget: planned = min(header limit, `limit-cap`), used = header limit - header remaining, left = planned - used. `github-token` defaults it to GitHub's documented GITHUB_TOKEN limit, so a higher header limit is headroom, never a target; the other pools have no cap and plan to their header limit. `plan` and `read-floor` are shares of the planned budget; `writer-pause` and `breaker` are shares of the header's limit, by its remaining; `content-creation` adds `per-minute` and `per-hour`. Their defaults are in `.github/actions/log-api-budget/rate-limits.json`, the only copy, which every merge hold list floors, so a change to a default waits for a person. The rate guard (`.github/actions/log-api-budget/rate-guard.mjs`, `planOf`) reads only `read-floor` and `limit-cap`, to warn once while calls continue, and stops only on the header's remaining and GitHub's refusals; `pace` (github-token, default 0.2) is the share of the header's limit, projected left at the window's reset from the burn rate so far, under which a timer-driven lane skips its even-numbered scheduled ticks, halving its cadence (`.github/actions/log-api-budget/pace.mjs`; the lost-run reconciler and pr-unblock-dispatch read it), so the timer lanes slow down before the breaker stops every lane; the others are thresholds meant for the hub's limits sampler, and nothing in this repository enforces them. resolve-config passes the block through as set. | report-outcome and the work-unblock sweep read them from resolve-config's output; pr-feedback-dispatch, pr-unblock-dispatch and work-resolve-issue's WIP cap check resolve the base diff --git a/docs/conventions/pr-pipeline/pr-pipeline.schema.json b/docs/conventions/pr-pipeline/pr-pipeline.schema.json index b472e6e0f78..3868c8c7353 100644 --- a/docs/conventions/pr-pipeline/pr-pipeline.schema.json +++ b/docs/conventions/pr-pipeline/pr-pipeline.schema.json @@ -370,6 +370,12 @@ "maximum": 1, "description": "Share of the header's limit left, by the header's remaining, at which the repository kill switch should be set. A threshold meant for the hub's limits sampler; the rate guard does not enforce it." }, + "pace": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Share of the header's limit, projected left at the window's reset from the burn rate so far, under which a timer-driven lane skips its even-numbered scheduled ticks (.github/actions/log-api-budget/pace.mjs), so the timer lanes slow to half cadence before the breaker stops every lane. 0 turns pacing off." + }, "read-floor": { "type": "number", "minimum": 0, From 6c445fd3b8916f07e2569dde6bf8c57dd3598818 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:03:23 -0400 Subject: [PATCH 4/5] fix(ci): hand the config's pace to the pacer, and log 0 calls when no verdicts download Neither paced workflow set RESOLVED_CONFIG, so pace.mjs read only rate-limits.json and limits.api github-token pace in pr-pipeline.yaml, 0 included, never reached it. Each now writes the base config's limits.api with yq, in resolve-config's output shape, and hands that file to the pacer; a failed read leaves the defaults. pr-review-main's record job: when the verdict download failed, the record step was skipped, no counter file was left, and the notice step's cat ended it red under bash -e. It now reads the file only when it exists, else 0. The test runs that step with -eo pipefail, as Actions does. Co-Authored-By: Claude Opus 5.5 --- .../perf-github-token-cuts-30e1480a.md | 3 +- .../log-api-budget/log-api-budget.test.mjs | 3 +- .github/actions/log-api-budget/pace.mjs | 3 +- .github/actions/log-api-budget/pace.test.mjs | 36 ++++++++++++++++++- .../maintenance-reconcile-lost-runs.yml | 16 ++++++--- .github/workflows/pr-review-main.yml | 4 ++- .github/workflows/pr-unblock-dispatch.yml | 17 ++++++--- docs/conventions/pr-pipeline/README.md | 2 +- 8 files changed, 70 insertions(+), 14 deletions(-) diff --git a/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md b/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md index e3de6e9bd32..dc7b935f721 100644 --- a/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md +++ b/.changes/pr-pipeline/perf-github-token-cuts-30e1480a.md @@ -4,7 +4,8 @@ bump: minor ### Added -- `limits.api` github-token `pace` (default 0.2, in rate-limits.json): the lost-run reconciler +- `limits.api` github-token `pace` (default 0.2, in rate-limits.json; set it in pr-pipeline.yaml, + 0 turns pacing off): the lost-run reconciler and pr-unblock-dispatch skip every other scheduled tick (the even run numbers) while the GITHUB_TOKEN pool's remaining at its reset, projected from the window's burn rate so far, is under that share of the header's limit, so the timer lanes slow to half cadence before the diff --git a/.github/actions/log-api-budget/log-api-budget.test.mjs b/.github/actions/log-api-budget/log-api-budget.test.mjs index f5420e6a68c..4a1a3388c09 100644 --- a/.github/actions/log-api-budget/log-api-budget.test.mjs +++ b/.github/actions/log-api-budget/log-api-budget.test.mjs @@ -829,7 +829,8 @@ test("pr-review-main's record job ends with its own api-calls notice in the emit const lib = new URL("api-budget.bash", import.meta.url).pathname; const emitted = (count) => { if (count !== undefined) writeFileSync(join(dir, "pr-review-main-calls"), `${count}\n`); - return execFileSync("bash", ["-c", run], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir } }); + // As Actions runs `shell: bash`: -e ends the step on any failing command. + return execFileSync("bash", ["--noprofile", "--norc", "-eo", "pipefail", "-c", run], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir } }); }; const shape = execFileSync("bash", ["-c", `. "${lib}"; api_budget_emit_calls github-token`], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir } }); assert.equal(emitted(), shape, "no count file is 0, as the emitter writes with no reading"); diff --git a/.github/actions/log-api-budget/pace.mjs b/.github/actions/log-api-budget/pace.mjs index 56682f98fe0..c4d74e0ccee 100644 --- a/.github/actions/log-api-budget/pace.mjs +++ b/.github/actions/log-api-budget/pace.mjs @@ -4,7 +4,8 @@ // schedule tick it reads the GITHUB_TOKEN pool once, projects the remaining // at the window's reset from the burn rate the window has shown so far, and // writes paced=true when that falls under `pace` of the header's limit -// (limits.api github-token pace, default in rate-limits.json). Only an +// (limits.api github-token pace: pr-pipeline.yaml, which each paced workflow +// hands over as RESOLVED_CONFIG, over rate-limits.json; 0 is off). Only an // even-numbered tick is paced, so a paced lane runs at half its cadence // rather than stopping: the lost-run reconciler, which re-runs the runs a // breaker trip stops, keeps working while the pool recovers. A paced tick diff --git a/.github/actions/log-api-budget/pace.test.mjs b/.github/actions/log-api-budget/pace.test.mjs index 5e96f1e5b4f..b82d8d6249b 100644 --- a/.github/actions/log-api-budget/pace.test.mjs +++ b/.github/actions/log-api-budget/pace.test.mjs @@ -4,7 +4,8 @@ // share the breaker is also stated in (pr-pipeline.schema.json api-pool). // Expected values are worked by hand from limit, remaining and reset. import assert from "node:assert/strict"; -import { readFileSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { test } from "node:test"; @@ -117,6 +118,32 @@ test("any other event, an unreadable answer or a failed read is never paced", as assert.match(failed.lines.join("\n"), /::warning::/); }); +// The step that hands the base config's limits.api to the pacer, in +// resolve-config's output shape, which apiLimits reads. +const READ_STEP = "Read the pace override from the base config"; +const READ_RUN = `yq -o=json '{"limits": {"api": (.limits.api // {})}}' .base/docs/conventions/pr-pipeline.yaml >"$RUNNER_TEMP/pace-config.json"`; + +// The documented off switch, limits.api github-token pace: 0 in +// pr-pipeline.yaml, through that step to main: a pool burning toward the +// breaker on an even tick runs, with no pace notice. +test("pace: 0 in the base config turns the pacer off", async () => { + const dir = tempDir("pace-off-"); + mkdirSync(join(dir, ".base/docs/conventions"), { recursive: true }); + const read = (yaml) => { + writeFileSync(join(dir, ".base/docs/conventions/pr-pipeline.yaml"), yaml); + const result = spawnSync("bash", ["--noprofile", "--norc", "-eo", "pipefail", "-c", READ_RUN], { cwd: dir, env: { PATH: process.env.PATH, RUNNER_TEMP: dir }, encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); + return join(dir, "pace-config.json"); + }; + const low = answer({ "x-ratelimit-limit": "5000", "x-ratelimit-remaining": "2900", "x-ratelimit-reset": String(NOW + 1800) }); + const off = await run({ RESOLVED_CONFIG: read("limits:\n api:\n github-token:\n pace: 0\n") }, low); + assert.equal(off.output, "paced=false\n"); + assert.deepEqual(off.lines, []); + // With no api entry the rate-limits.json default paces the same reading. + assert.equal((await run({ RESOLVED_CONFIG: read("limits:\n wip-cap: 12\n") }, low)).output, "paced=true\n"); + assert.equal(apiLimits({ resolvedPath: read("lanes: {}\n") })["github-token"].pace, 0.2, "no limits section"); +}); + // The timer-driven lanes that read the pacer: each reads it once on the // event, never fails on it, and gates every step that spends the pool. test("each paced timer lane reads the pacer before its work and gates every spending step on it", () => { @@ -131,6 +158,13 @@ test("each paced timer lane reads the pacer before its work and gates every spen const steps = block.split(/\n(?= {6}- name: )/).slice(1).map((s) => ({ name: /^ {6}- name: (.*)$/m.exec(s)[1], s: s.replace(/(\n *(#[^\n]*)?)+$/, "") })); const at = (name) => steps.findIndex((step) => step.name === name); const pace = steps[at("Pace the tick on the GITHUB_TOKEN burn rate")]?.s ?? ""; + // The base config's limits.api reaches the pacer, so its pace (0 for off) + // overrides rate-limits.json. + assert.ok(at(READ_STEP) >= 0 && at(READ_STEP) < at("Pace the tick on the GITHUB_TOKEN burn rate"), `${file}: the override is read before the pacer`); + assert.equal(steps[at(READ_STEP)].s.replace(/^[\s\S]*?\n {8}run: \|\n {10}/, ""), READ_RUN, file); + assert.match(steps[at(READ_STEP)].s, /\n {8}continue-on-error: true\n/, file); + assert.equal(/\n {8}if: (.*)\n/.exec(steps[at(READ_STEP)].s)?.[1], /\n {8}if: (.*)\n/.exec(pace)?.[1], `${file}: the read runs when the pacer does`); + assert.match(pace, /\n {10}RESOLVED_CONFIG: \$\{\{ runner\.temp \}\}\/pace-config\.json\n/, file); assert.match(pace, /\n {8}id: pace\n/, file); assert.match(pace, /\n {8}continue-on-error: true\n/, file); assert.match(pace, /\n {10}EVENT_NAME: \$\{\{ github\.event_name \}\}\n/, file); diff --git a/.github/workflows/maintenance-reconcile-lost-runs.yml b/.github/workflows/maintenance-reconcile-lost-runs.yml index 158eb81498f..7e373a73e4f 100644 --- a/.github/workflows/maintenance-reconcile-lost-runs.yml +++ b/.github/workflows/maintenance-reconcile-lost-runs.yml @@ -124,16 +124,24 @@ jobs: value: ${{ vars.CLAUDE_LANES_DISABLED }} repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + # The base config's limits.api, in resolve-config's output shape, for + # the pacer; a failed read leaves rate-limits.json's defaults. + - name: Read the pace override from the base config + continue-on-error: true + run: | + yq -o=json '{"limits": {"api": (.limits.api // {})}}' .base/docs/conventions/pr-pipeline.yaml >"$RUNNER_TEMP/pace-config.json" + # A schedule tick skips its work while the GITHUB_TOKEN pool's remaining # at reset, projected at this window's burn rate, is under limits.api - # github-token pace (rate-limits.json), on even run numbers only, so the - # timer lanes slow to half cadence before the breaker stops every lane. - # Any other event, or a read that fails, runs as before - # (.github/actions/log-api-budget/pace.mjs). + # github-token pace (pr-pipeline.yaml over rate-limits.json; 0 is off), + # on even run numbers only, so the timer lanes slow to half cadence + # before the breaker stops every lane. Any other event, or a read that + # fails, runs as before (.github/actions/log-api-budget/pace.mjs). - name: Pace the tick on the GITHUB_TOKEN burn rate id: pace continue-on-error: true env: + RESOLVED_CONFIG: ${{ runner.temp }}/pace-config.json GITHUB_TOKEN: ${{ github.token }} EVENT_NAME: ${{ github.event_name }} RUN_NUMBER: ${{ github.run_number }} diff --git a/.github/workflows/pr-review-main.yml b/.github/workflows/pr-review-main.yml index d48884e37c5..347e1b45157 100644 --- a/.github/workflows/pr-review-main.yml +++ b/.github/workflows/pr-review-main.yml @@ -1497,6 +1497,8 @@ jobs: - name: Log the API calls at the end if: always() run: | - calls=$(cat "$RUNNER_TEMP/pr-review-main-calls" 2>/dev/null) + # No file: the record step did not run (its download failed), so 0. + calls=0 + [[ -f $RUNNER_TEMP/pr-review-main-calls ]] && calls=$(<"$RUNNER_TEMP/pr-review-main-calls") [[ $calls =~ ^[0-9]{1,6}$ ]] || calls=0 echo "::notice title=api-calls::github-token=$calls not-modified=0 graphql=0" diff --git a/.github/workflows/pr-unblock-dispatch.yml b/.github/workflows/pr-unblock-dispatch.yml index 5101d81ef1f..cb327d1405a 100644 --- a/.github/workflows/pr-unblock-dispatch.yml +++ b/.github/workflows/pr-unblock-dispatch.yml @@ -138,17 +138,26 @@ jobs: echo "::error::${REASON:-untrusted-actor}: this run was not started by a listed trusted human" exit 1 + # The base config's limits.api, in resolve-config's output shape, for + # the pacer; a failed read leaves rate-limits.json's defaults. + - name: Read the pace override from the base config + if: steps.kill.outputs.proceed == 'true' + continue-on-error: true + run: | + yq -o=json '{"limits": {"api": (.limits.api // {})}}' .base/docs/conventions/pr-pipeline.yaml >"$RUNNER_TEMP/pace-config.json" + # A schedule tick skips its work while the GITHUB_TOKEN pool's remaining # at reset, projected at this window's burn rate, is under limits.api - # github-token pace (rate-limits.json), on even run numbers only, so the - # timer lanes slow to half cadence before the breaker stops every lane. - # Any other event, or a read that fails, runs as before - # (.github/actions/log-api-budget/pace.mjs). + # github-token pace (pr-pipeline.yaml over rate-limits.json; 0 is off), + # on even run numbers only, so the timer lanes slow to half cadence + # before the breaker stops every lane. Any other event, or a read that + # fails, runs as before (.github/actions/log-api-budget/pace.mjs). - name: Pace the tick on the GITHUB_TOKEN burn rate id: pace if: steps.kill.outputs.proceed == 'true' continue-on-error: true env: + RESOLVED_CONFIG: ${{ runner.temp }}/pace-config.json GITHUB_TOKEN: ${{ github.token }} EVENT_NAME: ${{ github.event_name }} RUN_NUMBER: ${{ github.run_number }} diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index ca7a28af640..c9ab1a16141 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -2837,7 +2837,7 @@ with its default. | `fix-docs-per-run` | 2 | fix-docs dispatches one pr-unblock-dispatch run sends; at most 20. | | `admission-factor` | 0.8 | Share of the lane's hourly merge rate admitted per hour for the queued cap ([WIP cap](#wip-cap)); greater than 0, at most 1. | | `admission-min-per-hour` | 1 | Admissions per hour the queued cap never falls below, so a day with no lane merges cannot hold admissions shut; 0 to 100, and 0 removes the floor. | -| `api` | schema | GitHub API margins per pool (`github-token`, `lanes-app`, `graphql`, `owner-token`, `content-creation`). `limit-cap` sets the pool's planned budget: planned = min(header limit, `limit-cap`), used = header limit - header remaining, left = planned - used. `github-token` defaults it to GitHub's documented GITHUB_TOKEN limit, so a higher header limit is headroom, never a target; the other pools have no cap and plan to their header limit. `plan` and `read-floor` are shares of the planned budget; `writer-pause` and `breaker` are shares of the header's limit, by its remaining; `content-creation` adds `per-minute` and `per-hour`. Their defaults are in `.github/actions/log-api-budget/rate-limits.json`, the only copy, which every merge hold list floors, so a change to a default waits for a person. The rate guard (`.github/actions/log-api-budget/rate-guard.mjs`, `planOf`) reads only `read-floor` and `limit-cap`, to warn once while calls continue, and stops only on the header's remaining and GitHub's refusals; `pace` (github-token, default 0.2) is the share of the header's limit, projected left at the window's reset from the burn rate so far, under which a timer-driven lane skips its even-numbered scheduled ticks, halving its cadence (`.github/actions/log-api-budget/pace.mjs`; the lost-run reconciler and pr-unblock-dispatch read it), so the timer lanes slow down before the breaker stops every lane; the others are thresholds meant for the hub's limits sampler, and nothing in this repository enforces them. resolve-config passes the block through as set. | +| `api` | schema | GitHub API margins per pool (`github-token`, `lanes-app`, `graphql`, `owner-token`, `content-creation`). `limit-cap` sets the pool's planned budget: planned = min(header limit, `limit-cap`), used = header limit - header remaining, left = planned - used. `github-token` defaults it to GitHub's documented GITHUB_TOKEN limit, so a higher header limit is headroom, never a target; the other pools have no cap and plan to their header limit. `plan` and `read-floor` are shares of the planned budget; `writer-pause` and `breaker` are shares of the header's limit, by its remaining; `content-creation` adds `per-minute` and `per-hour`. Their defaults are in `.github/actions/log-api-budget/rate-limits.json`, the only copy, which every merge hold list floors, so a change to a default waits for a person. The rate guard (`.github/actions/log-api-budget/rate-guard.mjs`, `planOf`) reads only `read-floor` and `limit-cap`, to warn once while calls continue, and stops only on the header's remaining and GitHub's refusals; `pace` (github-token, default 0.2) is the share of the header's limit, projected left at the window's reset from the burn rate so far, under which a timer-driven lane skips its even-numbered scheduled ticks, halving its cadence (`.github/actions/log-api-budget/pace.mjs`; the lost-run reconciler and pr-unblock-dispatch read it); 0 turns pacing off, so the timer lanes slow down before the breaker stops every lane; the others are thresholds meant for the hub's limits sampler, and nothing in this repository enforces them. resolve-config passes the block through as set. | report-outcome and the work-unblock sweep read them from resolve-config's output; pr-feedback-dispatch, pr-unblock-dispatch and work-resolve-issue's WIP cap check resolve the base From 402cee6cb39904d5cfa39dacdc5c16f37dcac273 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:49:21 -0400 Subject: [PATCH 5/5] docs(pr-pipeline): give pace its purpose before the off switch Reorder the limits.api pace sentence so "the timer lanes slow down before the breaker stops every lane" describes pace itself and "`0` turns pacing off" is its own clause, per the review thread. Co-Authored-By: Claude Opus 5.5 --- docs/conventions/pr-pipeline/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index c9ab1a16141..7c10da0ed5b 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -2837,7 +2837,7 @@ with its default. | `fix-docs-per-run` | 2 | fix-docs dispatches one pr-unblock-dispatch run sends; at most 20. | | `admission-factor` | 0.8 | Share of the lane's hourly merge rate admitted per hour for the queued cap ([WIP cap](#wip-cap)); greater than 0, at most 1. | | `admission-min-per-hour` | 1 | Admissions per hour the queued cap never falls below, so a day with no lane merges cannot hold admissions shut; 0 to 100, and 0 removes the floor. | -| `api` | schema | GitHub API margins per pool (`github-token`, `lanes-app`, `graphql`, `owner-token`, `content-creation`). `limit-cap` sets the pool's planned budget: planned = min(header limit, `limit-cap`), used = header limit - header remaining, left = planned - used. `github-token` defaults it to GitHub's documented GITHUB_TOKEN limit, so a higher header limit is headroom, never a target; the other pools have no cap and plan to their header limit. `plan` and `read-floor` are shares of the planned budget; `writer-pause` and `breaker` are shares of the header's limit, by its remaining; `content-creation` adds `per-minute` and `per-hour`. Their defaults are in `.github/actions/log-api-budget/rate-limits.json`, the only copy, which every merge hold list floors, so a change to a default waits for a person. The rate guard (`.github/actions/log-api-budget/rate-guard.mjs`, `planOf`) reads only `read-floor` and `limit-cap`, to warn once while calls continue, and stops only on the header's remaining and GitHub's refusals; `pace` (github-token, default 0.2) is the share of the header's limit, projected left at the window's reset from the burn rate so far, under which a timer-driven lane skips its even-numbered scheduled ticks, halving its cadence (`.github/actions/log-api-budget/pace.mjs`; the lost-run reconciler and pr-unblock-dispatch read it); 0 turns pacing off, so the timer lanes slow down before the breaker stops every lane; the others are thresholds meant for the hub's limits sampler, and nothing in this repository enforces them. resolve-config passes the block through as set. | +| `api` | schema | GitHub API margins per pool (`github-token`, `lanes-app`, `graphql`, `owner-token`, `content-creation`). `limit-cap` sets the pool's planned budget: planned = min(header limit, `limit-cap`), used = header limit - header remaining, left = planned - used. `github-token` defaults it to GitHub's documented GITHUB_TOKEN limit, so a higher header limit is headroom, never a target; the other pools have no cap and plan to their header limit. `plan` and `read-floor` are shares of the planned budget; `writer-pause` and `breaker` are shares of the header's limit, by its remaining; `content-creation` adds `per-minute` and `per-hour`. Their defaults are in `.github/actions/log-api-budget/rate-limits.json`, the only copy, which every merge hold list floors, so a change to a default waits for a person. The rate guard (`.github/actions/log-api-budget/rate-guard.mjs`, `planOf`) reads only `read-floor` and `limit-cap`, to warn once while calls continue, and stops only on the header's remaining and GitHub's refusals; `pace` (github-token, default 0.2) is the share of the header's limit, projected left at the window's reset from the burn rate so far, under which a timer-driven lane skips its even-numbered scheduled ticks, halving its cadence (`.github/actions/log-api-budget/pace.mjs`; the lost-run reconciler and pr-unblock-dispatch read it), so the timer lanes slow down before the breaker stops every lane; `0` turns pacing off; the others are thresholds meant for the hub's limits sampler, and nothing in this repository enforces them. resolve-config passes the block through as set. | report-outcome and the work-unblock sweep read them from resolve-config's output; pr-feedback-dispatch, pr-unblock-dispatch and work-resolve-issue's WIP cap check resolve the base