From b34f52c1586142cb4f8a2e971fce119ba5d1de4b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:56:48 +0000 Subject: [PATCH 1/2] ci: hold releases while a nightly Verify issue is open A failed nightly Verify opens an issue labelled nightly-verify. While such an issue is open, the release pull request checks fail Verify on every Release Please run, including runs that leave the pull request unchanged, and the nightly report turns off auto-merge on an open release pull request. A regression only the 5k and 10k-model jobs catch cannot ship. Closing the last open issue after a fix or a confirmed flake starts Release Please on main again, which re-checks the held pull request. --- .github/workflows/release-gate-recheck.yml | 40 ++++++++++++++++++ .github/workflows/release-please.yml | 49 +++++++++++++++++++++- .github/workflows/verify.yml | 8 ++++ CONTRIBUTING.md | 4 +- 4 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release-gate-recheck.yml diff --git a/.github/workflows/release-gate-recheck.yml b/.github/workflows/release-gate-recheck.yml new file mode 100644 index 0000000000..dca9f3e90c --- /dev/null +++ b/.github/workflows/release-gate-recheck.yml @@ -0,0 +1,40 @@ +name: Release gate re-check + +# Closing the last open nightly-verify issue starts Release Please on main when the release pull +# request is held, so its checks run again and it merges without a manual re-run. +on: + issues: + types: [closed] + +permissions: + actions: write + issues: read + pull-requests: read + statuses: read + +jobs: + recheck: + if: contains(github.event.issue.labels.*.name, 'nightly-verify') + runs-on: ubuntu-24.04 + steps: + - name: Start Release Please for a held release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + still_open="$(gh issue list --label nightly-verify --state open --json number --jq length)" + if [[ "${still_open}" != "0" ]]; then + echo "Another nightly-verify issue is still open; releases stay held" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + head_sha="$(gh pr list --state open --label "autorelease: pending" \ + --json headRefOid --jq '.[0].headRefOid // empty')" + [[ -n "${head_sha}" ]] || exit 0 + held="$(gh api "repos/${GH_REPO}/commits/${head_sha}/status" \ + --jq '[.statuses[] | select(.context == "Verify")][0].description // "" | startswith("Held by nightly Verify")')" + if [[ "${held}" != "true" ]]; then + echo "The release pull request is not held by nightly Verify" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + gh workflow run release-please.yml --ref main + echo "Started Release Please on main to re-check the held release" >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 50d8754844..c91ac3996b 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -9,6 +9,7 @@ on: permissions: actions: write contents: write + issues: read pull-requests: write statuses: write @@ -21,7 +22,7 @@ jobs: runs-on: ubuntu-24.04 outputs: prs_created: ${{ steps.release.outputs.prs_created }} - release_branch: ${{ steps.release_branch.outputs.name }} + release_branch: ${{ steps.release_branch.outputs.name || steps.open_release.outputs.branch }} release_pr: ${{ steps.release_branch.outputs.pr || steps.open_release.outputs.pr }} steps: - name: Create delivery token @@ -55,6 +56,8 @@ jobs: # A push that changes nothing in the release pull request, such as a docs commit, leaves it # as checked; the merge job re-enables auto-merge that an earlier run handed to this one. + # While a nightly-verify issue is open, or after one held this pull request, the release + # checks run again on its branch so the hold applies, or lifts, on every run. - name: Find the unchanged release pull request id: open_release if: steps.release.outputs.prs_created != 'true' && steps.release.outputs.release_created != 'true' @@ -64,6 +67,16 @@ jobs: release_pr="$(gh pr list --repo "${GITHUB_REPOSITORY}" --state open \ --label "autorelease: pending" --json number --jq '.[0].number // empty')" echo "pr=${release_pr}" >> "${GITHUB_OUTPUT}" + [[ -n "${release_pr}" ]] || exit 0 + read -r release_branch head_sha < <(gh pr view "${release_pr}" --repo "${GITHUB_REPOSITORY}" \ + --json headRefName,headRefOid --jq '"\(.headRefName) \(.headRefOid)"') + held="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${head_sha}/status" \ + --jq '[.statuses[] | select(.context == "Verify")][0].description // "" | startswith("Held by nightly Verify")')" + open="$(gh issue list --repo "${GITHUB_REPOSITORY}" --label nightly-verify --state open \ + --json number --jq length)" || open=unknown + if [[ "${held}" == "true" || "${open}" != "0" ]]; then + echo "branch=${release_branch}" >> "${GITHUB_OUTPUT}" + fi - name: Resolve release branch id: release_branch @@ -126,13 +139,45 @@ jobs: release-checks: name: Release pull request checks needs: release-please - if: needs.release-please.outputs.prs_created == 'true' + if: needs.release-please.outputs.release_branch != '' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: ref: ${{ needs.release-please.outputs.release_branch }} + # A failed nightly Verify opens an issue labelled nightly-verify. It holds every release + # until someone closes it after a fix or a confirmed flake; closing the last one starts + # Release Please again (release-gate-recheck.yml), which re-runs these checks. + - name: Check for open nightly Verify issues + env: + GH_TOKEN: ${{ github.token }} + RELEASE_PR: ${{ needs.release-please.outputs.release_pr }} + run: | + head_sha="$(git rev-parse HEAD)" + block() { + gh pr merge "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" --disable-auto 2>/dev/null || true + gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ + -f state=failure -f context=Verify -f "description=Held by nightly Verify: $1" \ + ${2:+-f "target_url=$2"} >/dev/null + echo "### Held by nightly Verify: $1" >> "${GITHUB_STEP_SUMMARY}" + exit 1 + } + if ! open="$(gh issue list --repo "${GITHUB_REPOSITORY}" --label nightly-verify \ + --state open --json url --jq '.[].url')"; then + block "could not list nightly-verify issues; rerun the failed Release Please job" + fi + if [[ -n "${open}" ]]; then + echo "${open}" >> "${GITHUB_STEP_SUMMARY}" + # An issue closed while the hold was posted found no hold to lift; lift it here. + ( block "close the open nightly-verify issue after a fix or a confirmed flake" \ + "$(head -n 1 <<< "${open}")" ) || true + still_open="$(gh issue list --repo "${GITHUB_REPOSITORY}" --label nightly-verify \ + --state open --json number --jq length)" || still_open=unknown + [[ "${still_open}" == "0" ]] || exit 1 + echo "The nightly-verify issue closed during this check; continuing" >> "${GITHUB_STEP_SUMMARY}" + fi + - name: Run release pull request checks env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 74401885ca..fd45e29958 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1407,6 +1407,7 @@ jobs: runs-on: ubuntu-24.04 permissions: issues: write + pull-requests: write steps: - name: Open or update the nightly Verify issue env: @@ -1433,3 +1434,10 @@ jobs: gh issue create --title "Nightly Verify failed on main" \ --label "${LABEL}" --body-file report.md fi + # The open issue holds releases (release-please.yml); stop a release pull request that + # already passed its checks from merging before Release Please runs again. + release_pr="$(gh pr list --state open --label "autorelease: pending" \ + --json number --jq '.[0].number // empty')" + if [[ -n "${release_pr}" ]]; then + gh pr merge "${release_pr}" --disable-auto 2>/dev/null || true + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 890b9a7327..79ff7fb2b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,9 @@ draft `full-verify` to run it on the pull request itself; until the merge queue (`SQLBUILD_MERGE_QUEUE`), a change that needs the heavy suites reports `Verify` only after that label's run. The 5,000 and 10,000-model timing jobs and every wheel platform run nightly on `main`, and a failed nightly run opens or -updates a `nightly-verify` issue. Each release builds and checks every wheel platform. +updates a `nightly-verify` issue. While that issue is open, release pull requests fail `Verify`; +closing it after a fix or a confirmed flake re-runs their checks. Each release builds and checks +every wheel platform. For focused iteration, run the narrowest relevant pytest target, for example: From 273883a0310d89901f5d161431cb9ec13e0e2318 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:39:30 +0000 Subject: [PATCH 2/2] ci: choose the full matrix from the pull request's current labels gh pr create --label can deliver labeled before opened. The later run cancels the earlier one, and the opened payload has no label, so the surviving run fell back to the fast core. Read the label and draft state from the pull request instead of the event payload. --- .github/workflows/verify.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index fd45e29958..9b2c046018 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -33,6 +33,9 @@ jobs: # A label other than `full-verify` changes nothing this workflow verifies. if: ${{ !((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify') }} runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read outputs: full: ${{ steps.depth.outputs.full }} partial: ${{ steps.depth.outputs.partial }} @@ -324,16 +327,23 @@ jobs: # every group before it merges, so a passing fast core reports as Verify and each pull # request runs the heavy suites once, in the queue. Unsetting the variable restores the old # behaviour. + # The label and draft state come from the pull request as it is now, not the event payload: + # `gh pr create --label` can deliver `labeled` before `opened`, and the later run cancels the + # earlier one, so the surviving run must still see the label. - name: Choose the fast core or the full matrix id: depth env: - FULL: >- - ${{ github.event_name != 'pull_request' - || (contains(github.event.pull_request.labels.*.name, 'full-verify') - && !github.event.pull_request.draft) }} + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} HEAVY_SELECTED: >- ${{ steps.scope.outputs.duckdb == 'true' || steps.scope.outputs.postgres == 'true' || steps.scope.outputs.clickhouse == 'true' || steps.scope.outputs.performance == 'true' || steps.scope.outputs.format_performance == 'true' || steps.scope.outputs.rules_performance == 'true' || steps.scope.outputs.wheels == 'true' || steps.scope.outputs.compiler_differential == 'true' || steps.scope.outputs.recovery == 'true' }} run: | + FULL=true + if [[ "${EVENT_NAME}" == "pull_request" ]]; then + FULL="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \ + --jq 'any(.labels[]; .name == "full-verify") and (.draft | not)')" + fi partial=false if [[ "${FULL}" != "true" && "${HEAVY_SELECTED}" == "true" ]]; then partial=true