diff --git a/.abcd/work/issues/open/iss-2610142326511964-macos-merge-queue-runs-slowed-to-34-40.md b/.abcd/work/issues/open/iss-2610142326511964-macos-merge-queue-runs-slowed-to-34-40.md new file mode 100644 index 0000000000..cdeed511f5 --- /dev/null +++ b/.abcd/work/issues/open/iss-2610142326511964-macos-merge-queue-runs-slowed-to-34-40.md @@ -0,0 +1,15 @@ +--- +schema_version: 1 +id: "iss-2610142326511964" +slug: "macos-merge-queue-runs-slowed-to-34-40" +severity: "minor" +category: "observation" +source: "agent-observation" +found_during: "raising the macOS check job limit to 60 minutes" +origin: researcher-authored +production_mode: hand-written +found_at: ".github/workflows/ci.yml" +remedy: "Profile the macOS race lane (the one race-enabled whole-module pass) and split or cache it before merge-queue runs reach the 60-minute job limit." +--- + +The macOS check job's merge-queue runs grew from 31-35 minutes to 34-40 minutes on 2026-10-14 (#990 took 40.0 minutes), close enough to the 45-minute job limit that the limit was raised to 60. The extra time is unexplained, and if it keeps growing the new limit will be reached too. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35f58f14ab..8edce6115c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,16 +265,20 @@ jobs: check: needs: changes if: ${{ !cancelled() }} - # The macOS leg's ceiling is equal to the merge queue's check response - # timeout (45 minutes, recorded in .abcd/work/rulesets/main-protection.json), - # which is the outer ceiling on a merge-group run: the queue fails the group - # when this required check has not concluded by then, so a job ceiling above - # it is unreachable there. Both are 45 under ruling Z (2026-09-28, the - # technical facilitator), because a 30-minute cap cancelled passing macOS - # runs of 23 to 30 minutes (iss-2609281514435020). Raising either again is - # an admin act on the live ruleset, left to the technical facilitator, and - # the remedy for a leg near the cap is a faster suite. The ubuntu leg runs - # well inside 30 minutes and keeps that ceiling. The race step's -timeout lifts go test's 10m + # The macOS leg's ceiling sits below the merge queue's check response + # timeout (90 minutes since 2026-10-11, recorded in + # .abcd/work/rulesets/main-protection.json), which is the outer ceiling on a + # merge-group run: the queue fails the group when this required check has + # not concluded by then, so a job ceiling above it would be unreachable + # there. Ruling Z (2026-09-28, the technical facilitator) raised the leg + # from 30 to 45 minutes, because a 30-minute cap cancelled passing macOS + # runs of 23 to 30 minutes (iss-2609281514435020). The technical + # facilitator raised it to 60 on 2026-10-15, when merge-queue macOS runs had + # grown from 31-35 to 34-40 minutes (iss-2610142326511964), so on this leg + # the job's own ceiling, not the queue's, is the binding one. Raising it + # again is the technical facilitator's call, and the remedy for a leg near + # the cap is a faster suite. The ubuntu leg runs well inside 30 minutes and + # keeps that ceiling. The race step's -timeout lifts go test's 10m # per-package default, so a slow package runs on to this ceiling instead of # failing at ten minutes. The step names its slowest packages first, so they # start with the step rather than behind every faster package; a hang in one @@ -283,7 +287,7 @@ jobs: # TestRaceLaneBudgetIsDeclaredAndFitsItsJob holds every leg at or below the # cap, and TestCheckJobWarnsBeforeTheQueueCap holds the warning step at the # end of the job below it. - timeout-minutes: ${{ matrix.os == 'macos-latest' && 45 || 30 }} + timeout-minutes: ${{ matrix.os == 'macos-latest' && 60 || 30 }} strategy: fail-fast: false matrix: @@ -397,9 +401,9 @@ jobs: # unrelated pull request from the merge queue (iss-2609260319483365). 20m # is about twice the slowest passing run, so a real hang still fails with # a goroutine dump where the enclosing ceiling leaves room for it: in - # release.yml's verify job it does, while this job's own ceiling, held to - # the merge queue's response cap, is the binding one on macOS (see the - # job header). The Makefile's preflight recipe and release.yml's verify + # release.yml's verify job it does, while this job's own ceiling, held + # below the merge queue's response cap, is the binding one on macOS (see + # the job header). The Makefile's preflight recipe and release.yml's verify # job carry the same flag, and so does the macOS step below; # TestRaceLaneBudgetIsDeclaredAndFitsItsJob holds all four. # @@ -517,24 +521,24 @@ jobs: # Warn before the cap cancels (ruling Z's cancel policy, 2026-09-28). A # macOS check that runs past WARN_AFTER_MINUTES is ten minutes from the - # merge queue's cap, and a cancellation there carries nothing but the + # leg's 60-minute ceiling, and a cancellation there carries nothing but the # cancellation. The rule for a cancelled check is to rerun it once; a # second cancellation for the same reason stops that pull request and # opens a speed lane, never another raise of the timeout. This warning # moves that signal ahead of the first cancellation: it is emitted at the # job's end, for a run that finished past the line, so a check that - # passed between 35 and 45 minutes says so before one is cancelled at + # passed between 50 and 60 minutes says so before one is cancelled at # the cap. It never fails the # job: `always()` runs it however the steps before it ended, and # `continue-on-error` holds even a broken script to a warning. - # TestCheckJobWarnsBeforeTheQueueCap pins the threshold below the leg's - # ceiling. + # TestCheckJobWarnsBeforeTheQueueCap pins the threshold ten minutes below + # the leg's ceiling. - name: Warn when the check nears the queue's cap if: always() && matrix.os == 'macos-latest' && github.event_name != 'pull_request' continue-on-error: true env: STARTED_AT: ${{ steps.started.outputs.epoch }} - WARN_AFTER_MINUTES: 35 + WARN_AFTER_MINUTES: 50 run: | case "${STARTED_AT:-}" in '' | *[!0-9]*) @@ -546,7 +550,7 @@ jobs: minutes=$((elapsed / 60)) echo "check (macos-latest) has run ${minutes}m $((elapsed % 60))s; the warning line is ${WARN_AFTER_MINUTES}m" if [ "$elapsed" -gt $((WARN_AFTER_MINUTES * 60)) ]; then - msg="check (macos-latest) ran ${minutes} minutes, past the ${WARN_AFTER_MINUTES}-minute warning line and close to the merge queue's cap. If it is cancelled at the cap, rerun it once; a second cancellation for the same reason stops the pull request: open a speed lane to make the suite faster, and do not raise the timeout." + msg="check (macos-latest) ran ${minutes} minutes, past the ${WARN_AFTER_MINUTES}-minute warning line and close to the job's 60-minute cap. If it is cancelled at the cap, rerun it once; a second cancellation for the same reason stops the pull request: open a speed lane to make the suite faster, and do not raise the timeout." echo "::warning title=macOS check near the queue's cap::${msg}" printf '### macOS check near the merge queue cap\n\n%s\n' "$msg" >>"$GITHUB_STEP_SUMMARY" fi diff --git a/internal/core/lint/racelanebudget_test.go b/internal/core/lint/racelanebudget_test.go index 13b73b32b3..8948e254c2 100644 --- a/internal/core/lint/racelanebudget_test.go +++ b/internal/core/lint/racelanebudget_test.go @@ -32,10 +32,11 @@ import ( // the event the lane was ejected on, and a job ceiling above it is unreachable // there: the queue gives up first, with no job failure to read. So no leg of // the check job may carry a timeout-minutes above the cap the ruleset mirror -// records. Under ruling Z (2026-09-28, the technical facilitator) the cap is 45 -// minutes and the macOS leg follows it, while the ubuntu leg keeps 30; the job -// declares that as one matrix expression, which checkLegTimeouts reads leg by -// leg. The step's -timeout still earns its place under that cap: it lifts the +// records. Ruling Z (2026-09-28, the technical facilitator) raised the macOS +// leg alone, while the ubuntu leg keeps 30; the job declares that as one matrix +// expression, which checkLegTimeouts reads leg by leg. The queue's cap is 90 +// minutes since 2026-10-11 and the macOS leg's is 60 since 2026-10-15, below +// it, so on that leg the job's own ceiling is the binding one. The step's -timeout still earns its place under that cap: it lifts the // ten-minute per-package default, so a slow package fails on the job's clock // rather than at ten minutes. Raising the queue's cap is an admin act on the // live ruleset, left to the technical facilitator; this test reads the mirror, @@ -122,6 +123,10 @@ func TestRaceLaneBudgetIsDeclaredAndFitsItsJob(t *testing.T) { "false. Raise the live ruleset's cap (an admin act) before the job's", where, leg.os, leg.minutes, queue, rulesetMirror) } + if leg.os == "macos-latest" && leg.minutes != checkMacOSCeilingMinutes*time.Minute { + t.Errorf("%s: the macos-latest leg's timeout-minutes is %s; want %dm, the ceiling "+ + "the technical facilitator set on 2026-10-15", where, leg.minutes, checkMacOSCeilingMinutes) + } // Ruling Z raised the macOS leg alone; every other leg keeps 30. if leg.os != "macos-latest" && leg.minutes != 30*time.Minute { t.Errorf("%s: the %s leg's timeout-minutes is %s; ruling Z (2026-09-28) raised only "+ @@ -147,11 +152,20 @@ func TestRaceLaneBudgetIsDeclaredAndFitsItsJob(t *testing.T) { } } +// checkMacOSCeilingMinutes is the macOS leg's timeout-minutes in ci.yml's check +// job: 45 under ruling Z (2026-09-28), 60 since 2026-10-15, when merge-queue +// macOS runs had grown from 31-35 to 34-40 minutes. It stays below the merge +// queue's check response timeout, which the budget test reads from the mirror. +const checkMacOSCeilingMinutes = 60 + +// checkWarnLeadMinutes is how far ahead of the macOS leg's ceiling the warning +// step fires: ruling Z's cancel policy (2026-09-28) set the line ten minutes +// under the cap, so a speed lane opens before anything is cancelled. +const checkWarnLeadMinutes = 10 + // checkWarnAfterMinutes is where the macOS leg of ci.yml's check job warns that -// it is nearing its ceiling: ruling Z's cancel policy (2026-09-28) sets it at 35 -// minutes, ten under the 45-minute cap, so a speed lane opens before the queue -// cancels anything. -const checkWarnAfterMinutes = 35 +// it is nearing its ceiling: checkWarnLeadMinutes under checkMacOSCeilingMinutes. +const checkWarnAfterMinutes = checkMacOSCeilingMinutes - checkWarnLeadMinutes // checkWarnStep is the name of that warning step in the check job. const checkWarnStep = "Warn when the check nears the queue's cap" @@ -182,8 +196,9 @@ func TestCheckJobWarnsBeforeTheQueueCap(t *testing.T) { } warn, _ := strconv.Atoi(m[1]) if warn != checkWarnAfterMinutes { - t.Errorf("%s step %q warns after %d minutes; the cancel policy of ruling Z sets %d", - where, checkWarnStep, warn, checkWarnAfterMinutes) + t.Errorf("%s step %q warns after %d minutes; want %d, %d under the macos-latest leg's "+ + "%d-minute ceiling (the cancel policy of ruling Z)", + where, checkWarnStep, warn, checkWarnAfterMinutes, checkWarnLeadMinutes, checkMacOSCeilingMinutes) } warnAfter := time.Duration(warn) * time.Minute for _, leg := range checkLegTimeouts(t, where, job) { @@ -191,6 +206,11 @@ func TestCheckJobWarnsBeforeTheQueueCap(t *testing.T) { t.Errorf("%s step %q warns after %s, not below the macos-latest leg's %s ceiling; "+ "the job is cancelled before the warning can fire", where, checkWarnStep, warnAfter, leg.minutes) } + if lead := time.Duration(checkWarnLeadMinutes) * time.Minute; leg.os == "macos-latest" && leg.minutes-warnAfter != lead { + t.Errorf("%s step %q warns after %s, %s before the macos-latest leg's %s ceiling; "+ + "the cancel policy of ruling Z puts the line %s before it", where, checkWarnStep, + warnAfter, leg.minutes-warnAfter, leg.minutes, lead) + } } if queue := mergeQueueResponseTimeout(t, root); warnAfter >= queue { t.Errorf("%s step %q warns after %s, not below the merge queue's %s cap in %s", diff --git a/internal/surface/cli/history_capture_cap_test.go b/internal/surface/cli/history_capture_cap_test.go index a22f768f76..bf2221b87f 100644 --- a/internal/surface/cli/history_capture_cap_test.go +++ b/internal/surface/cli/history_capture_cap_test.go @@ -30,8 +30,8 @@ import ( // standard-library code every git-execing test here runs under -race. So the // detector has nothing of this package's to watch, and instrumenting // those passes cost 125s against 6s uninstrumented on the same machine: a third -// of this package's race time, on the macOS leg whose job is held to the merge -// queue's 45-minute cap (iss-2609261924541555). The uninstrumented lane asserts +// of this package's race time, on the macOS leg whose job is held to a +// 60-minute ceiling (iss-2609261924541555). The uninstrumented lane asserts // the same cap on both CI legs. func TestHistoryCaptureAcceptsWhatTheHooksAccept(t *testing.T) { if raceEnabled {