From 5097e83b84a2de6d48eef91123af60fc8c95ecb6 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:27:40 +0100 Subject: [PATCH 1/2] chore: capture the macOS merge-queue slowdown to 34-40 minutes Refs: iss-2610142326511964 Assisted-by: Claude:claude-opus-5-5 --- ...1964-macos-merge-queue-runs-slowed-to-34-40.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .abcd/work/issues/open/iss-2610142326511964-macos-merge-queue-runs-slowed-to-34-40.md diff --git a/.abcd/work/issues/open/iss-2610142326511964-macos-merge-queue-runs-slowed-to-34-40.md b/.abcd/work/issues/open/iss-2610142326511964-macos-merge-queue-runs-slowed-to-34-40.md new file mode 100644 index 0000000000..cdeed511f5 --- /dev/null +++ b/.abcd/work/issues/open/iss-2610142326511964-macos-merge-queue-runs-slowed-to-34-40.md @@ -0,0 +1,15 @@ +--- +schema_version: 1 +id: "iss-2610142326511964" +slug: "macos-merge-queue-runs-slowed-to-34-40" +severity: "minor" +category: "observation" +source: "agent-observation" +found_during: "raising the macOS check job limit to 60 minutes" +origin: researcher-authored +production_mode: hand-written +found_at: ".github/workflows/ci.yml" +remedy: "Profile the macOS race lane (the one race-enabled whole-module pass) and split or cache it before merge-queue runs reach the 60-minute job limit." +--- + +The macOS check job's merge-queue runs grew from 31-35 minutes to 34-40 minutes on 2026-10-14 (#990 took 40.0 minutes), close enough to the 45-minute job limit that the limit was raised to 60. The extra time is unexplained, and if it keeps growing the new limit will be reached too. From 3d3d11e8a72b03191c13510cb89d09b6d832f3c1 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:27:42 +0100 Subject: [PATCH 2/2] ci: raise the macOS check job's limit from 45 to 60 minutes Merge-queue macOS runs grew from 31-35 to 34-40 minutes on 2026-10-14 (#990 took 40.0), close to the 45-minute limit. The queue's check response timeout is already 90 (ruleset 19969675, mirrored in .abcd/work/rulesets/main-protection.json), so the job's own limit is the binding one and 60 stays under it. The warning line moves with the limit, from 35 to 50, keeping ruling Z's ten-minute lead. The budget test now pins the macOS ceiling at 60 and the warning test holds the line ten minutes under it. The ci.yml comments that said the queue waits 45 minutes are corrected. Refs: iss-2610142326511964 Assisted-by: Claude:claude-opus-5-5 --- .github/workflows/ci.yml | 44 ++++++++++--------- internal/core/lint/racelanebudget_test.go | 40 ++++++++++++----- .../surface/cli/history_capture_cap_test.go | 4 +- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b43e51a00c..e05cb58301 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,16 +265,20 @@ jobs: check: needs: changes if: ${{ !cancelled() }} - # The macOS leg's ceiling is equal to the merge queue's check response - # timeout (45 minutes, recorded in .abcd/work/rulesets/main-protection.json), - # which is the outer ceiling on a merge-group run: the queue fails the group - # when this required check has not concluded by then, so a job ceiling above - # it is unreachable there. Both are 45 under ruling Z (2026-09-28, the - # technical facilitator), because a 30-minute cap cancelled passing macOS - # runs of 23 to 30 minutes (iss-2609281514435020). Raising either again is - # an admin act on the live ruleset, left to the technical facilitator, and - # the remedy for a leg near the cap is a faster suite. The ubuntu leg runs - # well inside 30 minutes and keeps that ceiling. The race step's -timeout lifts go test's 10m + # The macOS leg's ceiling sits below the merge queue's check response + # timeout (90 minutes since 2026-10-11, recorded in + # .abcd/work/rulesets/main-protection.json), which is the outer ceiling on a + # merge-group run: the queue fails the group when this required check has + # not concluded by then, so a job ceiling above it would be unreachable + # there. Ruling Z (2026-09-28, the technical facilitator) raised the leg + # from 30 to 45 minutes, because a 30-minute cap cancelled passing macOS + # runs of 23 to 30 minutes (iss-2609281514435020). The technical + # facilitator raised it to 60 on 2026-10-15, when merge-queue macOS runs had + # grown from 31-35 to 34-40 minutes (iss-2610142326511964), so on this leg + # the job's own ceiling, not the queue's, is the binding one. Raising it + # again is the technical facilitator's call, and the remedy for a leg near + # the cap is a faster suite. The ubuntu leg runs well inside 30 minutes and + # keeps that ceiling. The race step's -timeout lifts go test's 10m # per-package default, so a slow package runs on to this ceiling instead of # failing at ten minutes. The step names its slowest packages first, so they # start with the step rather than behind every faster package; a hang in one @@ -283,7 +287,7 @@ jobs: # TestRaceLaneBudgetIsDeclaredAndFitsItsJob holds every leg at or below the # cap, and TestCheckJobWarnsBeforeTheQueueCap holds the warning step at the # end of the job below it. - timeout-minutes: ${{ matrix.os == 'macos-latest' && 45 || 30 }} + timeout-minutes: ${{ matrix.os == 'macos-latest' && 60 || 30 }} strategy: fail-fast: false matrix: @@ -390,9 +394,9 @@ jobs: # unrelated pull request from the merge queue (iss-2609260319483365). 20m # is about twice the slowest passing run, so a real hang still fails with # a goroutine dump where the enclosing ceiling leaves room for it: in - # release.yml's verify job it does, while this job's own ceiling, held to - # the merge queue's response cap, is the binding one on macOS (see the - # job header). The Makefile's preflight recipe and release.yml's verify + # release.yml's verify job it does, while this job's own ceiling, held + # below the merge queue's response cap, is the binding one on macOS (see + # the job header). The Makefile's preflight recipe and release.yml's verify # job carry the same flag, and so does the macOS step below; # TestRaceLaneBudgetIsDeclaredAndFitsItsJob holds all four. # @@ -506,24 +510,24 @@ jobs: # Warn before the cap cancels (ruling Z's cancel policy, 2026-09-28). A # macOS check that runs past WARN_AFTER_MINUTES is ten minutes from the - # merge queue's cap, and a cancellation there carries nothing but the + # leg's 60-minute ceiling, and a cancellation there carries nothing but the # cancellation. The rule for a cancelled check is to rerun it once; a # second cancellation for the same reason stops that pull request and # opens a speed lane, never another raise of the timeout. This warning # moves that signal ahead of the first cancellation: it is emitted at the # job's end, for a run that finished past the line, so a check that - # passed between 35 and 45 minutes says so before one is cancelled at + # passed between 50 and 60 minutes says so before one is cancelled at # the cap. It never fails the # job: `always()` runs it however the steps before it ended, and # `continue-on-error` holds even a broken script to a warning. - # TestCheckJobWarnsBeforeTheQueueCap pins the threshold below the leg's - # ceiling. + # TestCheckJobWarnsBeforeTheQueueCap pins the threshold ten minutes below + # the leg's ceiling. - name: Warn when the check nears the queue's cap if: always() && matrix.os == 'macos-latest' && github.event_name != 'pull_request' continue-on-error: true env: STARTED_AT: ${{ steps.started.outputs.epoch }} - WARN_AFTER_MINUTES: 35 + WARN_AFTER_MINUTES: 50 run: | case "${STARTED_AT:-}" in '' | *[!0-9]*) @@ -535,7 +539,7 @@ jobs: minutes=$((elapsed / 60)) echo "check (macos-latest) has run ${minutes}m $((elapsed % 60))s; the warning line is ${WARN_AFTER_MINUTES}m" if [ "$elapsed" -gt $((WARN_AFTER_MINUTES * 60)) ]; then - msg="check (macos-latest) ran ${minutes} minutes, past the ${WARN_AFTER_MINUTES}-minute warning line and close to the merge queue's cap. If it is cancelled at the cap, rerun it once; a second cancellation for the same reason stops the pull request: open a speed lane to make the suite faster, and do not raise the timeout." + msg="check (macos-latest) ran ${minutes} minutes, past the ${WARN_AFTER_MINUTES}-minute warning line and close to the job's 60-minute cap. If it is cancelled at the cap, rerun it once; a second cancellation for the same reason stops the pull request: open a speed lane to make the suite faster, and do not raise the timeout." echo "::warning title=macOS check near the queue's cap::${msg}" printf '### macOS check near the merge queue cap\n\n%s\n' "$msg" >>"$GITHUB_STEP_SUMMARY" fi diff --git a/internal/core/lint/racelanebudget_test.go b/internal/core/lint/racelanebudget_test.go index efaf167da3..ef32aa40e3 100644 --- a/internal/core/lint/racelanebudget_test.go +++ b/internal/core/lint/racelanebudget_test.go @@ -32,10 +32,11 @@ import ( // the event the lane was ejected on, and a job ceiling above it is unreachable // there: the queue gives up first, with no job failure to read. So no leg of // the check job may carry a timeout-minutes above the cap the ruleset mirror -// records. Under ruling Z (2026-09-28, the technical facilitator) the cap is 45 -// minutes and the macOS leg follows it, while the ubuntu leg keeps 30; the job -// declares that as one matrix expression, which checkLegTimeouts reads leg by -// leg. The step's -timeout still earns its place under that cap: it lifts the +// records. Ruling Z (2026-09-28, the technical facilitator) raised the macOS +// leg alone, while the ubuntu leg keeps 30; the job declares that as one matrix +// expression, which checkLegTimeouts reads leg by leg. The queue's cap is 90 +// minutes since 2026-10-11 and the macOS leg's is 60 since 2026-10-15, below +// it, so on that leg the job's own ceiling is the binding one. The step's -timeout still earns its place under that cap: it lifts the // ten-minute per-package default, so a slow package fails on the job's clock // rather than at ten minutes. Raising the queue's cap is an admin act on the // live ruleset, left to the technical facilitator; this test reads the mirror, @@ -122,6 +123,10 @@ func TestRaceLaneBudgetIsDeclaredAndFitsItsJob(t *testing.T) { "false. Raise the live ruleset's cap (an admin act) before the job's", where, leg.os, leg.minutes, queue, rulesetMirror) } + if leg.os == "macos-latest" && leg.minutes != checkMacOSCeilingMinutes*time.Minute { + t.Errorf("%s: the macos-latest leg's timeout-minutes is %s; want %dm, the ceiling "+ + "the technical facilitator set on 2026-10-15", where, leg.minutes, checkMacOSCeilingMinutes) + } // Ruling Z raised the macOS leg alone; every other leg keeps 30. if leg.os != "macos-latest" && leg.minutes != 30*time.Minute { t.Errorf("%s: the %s leg's timeout-minutes is %s; ruling Z (2026-09-28) raised only "+ @@ -147,11 +152,20 @@ func TestRaceLaneBudgetIsDeclaredAndFitsItsJob(t *testing.T) { } } +// checkMacOSCeilingMinutes is the macOS leg's timeout-minutes in ci.yml's check +// job: 45 under ruling Z (2026-09-28), 60 since 2026-10-15, when merge-queue +// macOS runs had grown from 31-35 to 34-40 minutes. It stays below the merge +// queue's check response timeout, which the budget test reads from the mirror. +const checkMacOSCeilingMinutes = 60 + +// checkWarnLeadMinutes is how far ahead of the macOS leg's ceiling the warning +// step fires: ruling Z's cancel policy (2026-09-28) set the line ten minutes +// under the cap, so a speed lane opens before anything is cancelled. +const checkWarnLeadMinutes = 10 + // checkWarnAfterMinutes is where the macOS leg of ci.yml's check job warns that -// it is nearing its ceiling: ruling Z's cancel policy (2026-09-28) sets it at 35 -// minutes, ten under the 45-minute cap, so a speed lane opens before the queue -// cancels anything. -const checkWarnAfterMinutes = 35 +// it is nearing its ceiling: checkWarnLeadMinutes under checkMacOSCeilingMinutes. +const checkWarnAfterMinutes = checkMacOSCeilingMinutes - checkWarnLeadMinutes // checkWarnStep is the name of that warning step in the check job. const checkWarnStep = "Warn when the check nears the queue's cap" @@ -182,8 +196,9 @@ func TestCheckJobWarnsBeforeTheQueueCap(t *testing.T) { } warn, _ := strconv.Atoi(m[1]) if warn != checkWarnAfterMinutes { - t.Errorf("%s step %q warns after %d minutes; the cancel policy of ruling Z sets %d", - where, checkWarnStep, warn, checkWarnAfterMinutes) + t.Errorf("%s step %q warns after %d minutes; want %d, %d under the macos-latest leg's "+ + "%d-minute ceiling (the cancel policy of ruling Z)", + where, checkWarnStep, warn, checkWarnAfterMinutes, checkWarnLeadMinutes, checkMacOSCeilingMinutes) } warnAfter := time.Duration(warn) * time.Minute for _, leg := range checkLegTimeouts(t, where, job) { @@ -191,6 +206,11 @@ func TestCheckJobWarnsBeforeTheQueueCap(t *testing.T) { t.Errorf("%s step %q warns after %s, not below the macos-latest leg's %s ceiling; "+ "the job is cancelled before the warning can fire", where, checkWarnStep, warnAfter, leg.minutes) } + if lead := time.Duration(checkWarnLeadMinutes) * time.Minute; leg.os == "macos-latest" && leg.minutes-warnAfter != lead { + t.Errorf("%s step %q warns after %s, %s before the macos-latest leg's %s ceiling; "+ + "the cancel policy of ruling Z puts the line %s before it", where, checkWarnStep, + warnAfter, leg.minutes-warnAfter, leg.minutes, lead) + } } if queue := mergeQueueResponseTimeout(t, root); warnAfter >= queue { t.Errorf("%s step %q warns after %s, not below the merge queue's %s cap in %s", diff --git a/internal/surface/cli/history_capture_cap_test.go b/internal/surface/cli/history_capture_cap_test.go index a22f768f76..bf2221b87f 100644 --- a/internal/surface/cli/history_capture_cap_test.go +++ b/internal/surface/cli/history_capture_cap_test.go @@ -30,8 +30,8 @@ import ( // standard-library code every git-execing test here runs under -race. So the // detector has nothing of this package's to watch, and instrumenting // those passes cost 125s against 6s uninstrumented on the same machine: a third -// of this package's race time, on the macOS leg whose job is held to the merge -// queue's 45-minute cap (iss-2609261924541555). The uninstrumented lane asserts +// of this package's race time, on the macOS leg whose job is held to a +// 60-minute ceiling (iss-2609261924541555). The uninstrumented lane asserts // the same cap on both CI legs. func TestHistoryCaptureAcceptsWhatTheHooksAccept(t *testing.T) { if raceEnabled {