diff --git a/.github/workflows/release-gate-recheck.yml b/.github/workflows/release-gate-recheck.yml new file mode 100644 index 0000000000..dca9f3e90c --- /dev/null +++ b/.github/workflows/release-gate-recheck.yml @@ -0,0 +1,40 @@ +name: Release gate re-check + +# Closing the last open nightly-verify issue starts Release Please on main when the release pull +# request is held, so its checks run again and it merges without a manual re-run. +on: + issues: + types: [closed] + +permissions: + actions: write + issues: read + pull-requests: read + statuses: read + +jobs: + recheck: + if: contains(github.event.issue.labels.*.name, 'nightly-verify') + runs-on: ubuntu-24.04 + steps: + - name: Start Release Please for a held release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + still_open="$(gh issue list --label nightly-verify --state open --json number --jq length)" + if [[ "${still_open}" != "0" ]]; then + echo "Another nightly-verify issue is still open; releases stay held" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + head_sha="$(gh pr list --state open --label "autorelease: pending" \ + --json headRefOid --jq '.[0].headRefOid // empty')" + [[ -n "${head_sha}" ]] || exit 0 + held="$(gh api "repos/${GH_REPO}/commits/${head_sha}/status" \ + --jq '[.statuses[] | select(.context == "Verify")][0].description // "" | startswith("Held by nightly Verify")')" + if [[ "${held}" != "true" ]]; then + echo "The release pull request is not held by nightly Verify" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + gh workflow run release-please.yml --ref main + echo "Started Release Please on main to re-check the held release" >> "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 50d8754844..c91ac3996b 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -9,6 +9,7 @@ on: permissions: actions: write contents: write + issues: read pull-requests: write statuses: write @@ -21,7 +22,7 @@ jobs: runs-on: ubuntu-24.04 outputs: prs_created: ${{ steps.release.outputs.prs_created }} - release_branch: ${{ steps.release_branch.outputs.name }} + release_branch: ${{ steps.release_branch.outputs.name || steps.open_release.outputs.branch }} release_pr: ${{ steps.release_branch.outputs.pr || steps.open_release.outputs.pr }} steps: - name: Create delivery token @@ -55,6 +56,8 @@ jobs: # A push that changes nothing in the release pull request, such as a docs commit, leaves it # as checked; the merge job re-enables auto-merge that an earlier run handed to this one. + # While a nightly-verify issue is open, or after one held this pull request, the release + # checks run again on its branch so the hold applies, or lifts, on every run. - name: Find the unchanged release pull request id: open_release if: steps.release.outputs.prs_created != 'true' && steps.release.outputs.release_created != 'true' @@ -64,6 +67,16 @@ jobs: release_pr="$(gh pr list --repo "${GITHUB_REPOSITORY}" --state open \ --label "autorelease: pending" --json number --jq '.[0].number // empty')" echo "pr=${release_pr}" >> "${GITHUB_OUTPUT}" + [[ -n "${release_pr}" ]] || exit 0 + read -r release_branch head_sha < <(gh pr view "${release_pr}" --repo "${GITHUB_REPOSITORY}" \ + --json headRefName,headRefOid --jq '"\(.headRefName) \(.headRefOid)"') + held="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${head_sha}/status" \ + --jq '[.statuses[] | select(.context == "Verify")][0].description // "" | startswith("Held by nightly Verify")')" + open="$(gh issue list --repo "${GITHUB_REPOSITORY}" --label nightly-verify --state open \ + --json number --jq length)" || open=unknown + if [[ "${held}" == "true" || "${open}" != "0" ]]; then + echo "branch=${release_branch}" >> "${GITHUB_OUTPUT}" + fi - name: Resolve release branch id: release_branch @@ -126,13 +139,45 @@ jobs: release-checks: name: Release pull request checks needs: release-please - if: needs.release-please.outputs.prs_created == 'true' + if: needs.release-please.outputs.release_branch != '' runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: ref: ${{ needs.release-please.outputs.release_branch }} + # A failed nightly Verify opens an issue labelled nightly-verify. It holds every release + # until someone closes it after a fix or a confirmed flake; closing the last one starts + # Release Please again (release-gate-recheck.yml), which re-runs these checks. + - name: Check for open nightly Verify issues + env: + GH_TOKEN: ${{ github.token }} + RELEASE_PR: ${{ needs.release-please.outputs.release_pr }} + run: | + head_sha="$(git rev-parse HEAD)" + block() { + gh pr merge "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" --disable-auto 2>/dev/null || true + gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ + -f state=failure -f context=Verify -f "description=Held by nightly Verify: $1" \ + ${2:+-f "target_url=$2"} >/dev/null + echo "### Held by nightly Verify: $1" >> "${GITHUB_STEP_SUMMARY}" + exit 1 + } + if ! open="$(gh issue list --repo "${GITHUB_REPOSITORY}" --label nightly-verify \ + --state open --json url --jq '.[].url')"; then + block "could not list nightly-verify issues; rerun the failed Release Please job" + fi + if [[ -n "${open}" ]]; then + echo "${open}" >> "${GITHUB_STEP_SUMMARY}" + # An issue closed while the hold was posted found no hold to lift; lift it here. + ( block "close the open nightly-verify issue after a fix or a confirmed flake" \ + "$(head -n 1 <<< "${open}")" ) || true + still_open="$(gh issue list --repo "${GITHUB_REPOSITORY}" --label nightly-verify \ + --state open --json number --jq length)" || still_open=unknown + [[ "${still_open}" == "0" ]] || exit 1 + echo "The nightly-verify issue closed during this check; continuing" >> "${GITHUB_STEP_SUMMARY}" + fi + - name: Run release pull request checks env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 74401885ca..9b2c046018 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -33,6 +33,9 @@ jobs: # A label other than `full-verify` changes nothing this workflow verifies. if: ${{ !((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify') }} runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read outputs: full: ${{ steps.depth.outputs.full }} partial: ${{ steps.depth.outputs.partial }} @@ -324,16 +327,23 @@ jobs: # every group before it merges, so a passing fast core reports as Verify and each pull # request runs the heavy suites once, in the queue. Unsetting the variable restores the old # behaviour. + # The label and draft state come from the pull request as it is now, not the event payload: + # `gh pr create --label` can deliver `labeled` before `opened`, and the later run cancels the + # earlier one, so the surviving run must still see the label. - name: Choose the fast core or the full matrix id: depth env: - FULL: >- - ${{ github.event_name != 'pull_request' - || (contains(github.event.pull_request.labels.*.name, 'full-verify') - && !github.event.pull_request.draft) }} + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} HEAVY_SELECTED: >- ${{ steps.scope.outputs.duckdb == 'true' || steps.scope.outputs.postgres == 'true' || steps.scope.outputs.clickhouse == 'true' || steps.scope.outputs.performance == 'true' || steps.scope.outputs.format_performance == 'true' || steps.scope.outputs.rules_performance == 'true' || steps.scope.outputs.wheels == 'true' || steps.scope.outputs.compiler_differential == 'true' || steps.scope.outputs.recovery == 'true' }} run: | + FULL=true + if [[ "${EVENT_NAME}" == "pull_request" ]]; then + FULL="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \ + --jq 'any(.labels[]; .name == "full-verify") and (.draft | not)')" + fi partial=false if [[ "${FULL}" != "true" && "${HEAVY_SELECTED}" == "true" ]]; then partial=true @@ -1407,6 +1417,7 @@ jobs: runs-on: ubuntu-24.04 permissions: issues: write + pull-requests: write steps: - name: Open or update the nightly Verify issue env: @@ -1433,3 +1444,10 @@ jobs: gh issue create --title "Nightly Verify failed on main" \ --label "${LABEL}" --body-file report.md fi + # The open issue holds releases (release-please.yml); stop a release pull request that + # already passed its checks from merging before Release Please runs again. + release_pr="$(gh pr list --state open --label "autorelease: pending" \ + --json number --jq '.[0].number // empty')" + if [[ -n "${release_pr}" ]]; then + gh pr merge "${release_pr}" --disable-auto 2>/dev/null || true + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 890b9a7327..79ff7fb2b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,9 @@ draft `full-verify` to run it on the pull request itself; until the merge queue (`SQLBUILD_MERGE_QUEUE`), a change that needs the heavy suites reports `Verify` only after that label's run. The 5,000 and 10,000-model timing jobs and every wheel platform run nightly on `main`, and a failed nightly run opens or -updates a `nightly-verify` issue. Each release builds and checks every wheel platform. +updates a `nightly-verify` issue. While that issue is open, release pull requests fail `Verify`; +closing it after a fix or a confirmed flake re-runs their checks. Each release builds and checks +every wheel platform. For focused iteration, run the narrowest relevant pytest target, for example: