From 84a4bc2e4161104daf9ff0640c54968c09de8b52 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:38:45 +0000 Subject: [PATCH 1/2] wip(ci): build only the linux-x86_64 wheel on pull requests and merge groups Work in progress, first of five planned CI speed-ups. Not yet opened as a pull request. build-distributions.yml takes a platforms input; its source job filters the one platform list into the wheel matrix and rejects unknown names. Verify passes linux-x86_64 on pull requests and merge groups, which Compose needs; manual runs and Release Distributions still build every platform. Validated: the filter script run locally for an empty selection (all five), one platform, two platforms and an unknown name (fails); both workflows parse as YAML; make check-release-gate passes. Not yet run in GitHub Actions. --- .github/workflows/build-distributions.yml | 75 ++++++++++++++--------- .github/workflows/verify.yml | 5 ++ 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index a14f9e02dd..3a5c993e5c 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -46,6 +46,13 @@ on: required: false type: boolean default: false + platforms: + description: >- + Space-separated wheel platforms to build, such as linux-x86_64; empty builds every + platform. Releases and the nightly Verify build every platform. + required: false + type: string + default: "" outputs: sha: description: "Commit the distributions were built from" @@ -61,7 +68,45 @@ jobs: outputs: sha: ${{ steps.source.outputs.sha }} tree: ${{ steps.source.outputs.tree }} + platforms: ${{ steps.platforms.outputs.matrix }} steps: + - name: Select the wheel platforms + id: platforms + env: + SELECTED: ${{ inputs.platforms }} + WRITER: ${{ inputs.dependency-cache-writer }} + # dependency-cache marks the slowest platforms, whose dist builds restore compiled + # dependencies. Linux builds run in maturin's manylinux container, where the host cache + # does not apply. A dependency cache writer builds only the marked platforms. + ALL: >- + [ + {"name": "linux-x86_64", "runner": "ubuntu-24.04", "target": "x86_64", + "dependency-cache": "none"}, + {"name": "linux-aarch64", "runner": "ubuntu-24.04-arm", "target": "aarch64", + "dependency-cache": "none"}, + {"name": "macos-x86_64", "runner": "macos-15", "target": "x86_64", + "python-architecture": "x64", "smoke-arch": "x86_64", "dependency-cache": "restore"}, + {"name": "macos-arm64", "runner": "macos-15", "target": "aarch64", + "dependency-cache": "restore"}, + {"name": "windows-x86_64", "runner": "windows-2025", "target": "x86_64", + "dependency-cache": "none"} + ] + run: | + matrix="$(jq -c --arg selected "${SELECTED}" --arg writer "${WRITER}" ' + ($selected | split(" ") | map(select(. != ""))) as $names + | if ($names | length) == 0 then . else map(select(.name as $n | $names | index($n))) end + | if $writer == "true" then map(select(.["dependency-cache"] != "none")) else . end + ' <<< "${ALL}")" + unknown="$(jq -rn --arg selected "${SELECTED}" --argjson all "${ALL}" ' + $selected | split(" ") | map(select(. != "" and (. as $n | $all | map(.name) | index($n) | not))) + | join(" ")')" + if [[ -n "${unknown}" ]]; then + echo "Unknown wheel platforms: ${unknown}" >&2 + exit 1 + fi + echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}" + echo "Wheel platforms: $(jq -r 'map(.name) | join(", ")' <<< "${matrix}")" \ + >> "${GITHUB_STEP_SUMMARY}" - uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} @@ -106,35 +151,7 @@ jobs: strategy: fail-fast: false matrix: - # dependency-cache marks the slowest platforms, whose dist builds restore compiled - # dependencies. Linux builds run in maturin's manylinux container, where the host cache - # does not apply. - platform: - - name: linux-x86_64 - runner: ubuntu-24.04 - target: x86_64 - dependency-cache: none - - name: linux-aarch64 - runner: ubuntu-24.04-arm - target: aarch64 - dependency-cache: none - - name: macos-x86_64 - runner: macos-15 - target: x86_64 - python-architecture: x64 - smoke-arch: x86_64 - dependency-cache: restore - - name: macos-arm64 - runner: macos-15 - target: aarch64 - dependency-cache: restore - - name: windows-x86_64 - runner: windows-2025 - target: x86_64 - dependency-cache: none - exclude: - - platform: - dependency-cache: ${{ inputs.dependency-cache-writer && 'none' || 'never' }} + platform: ${{ fromJSON(needs.source.outputs.platforms) }} steps: - uses: actions/checkout@v4 if: inputs.prebuilt-run-id == '' diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 22711433e8..1170454220 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1150,6 +1150,11 @@ jobs: retention-days: 3 # Release Distributions builds and smoke tests the slower dist profile for every release. cargo-profile: release + # Pull requests and merge groups build the wheel Compose installs; manual runs build every + # platform, and Release Distributions builds every platform for each release. + platforms: >- + ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') + && 'linux-x86_64' || '' }} compose: name: Packaged Compose lifecycle From 636f45ea525330341726e5741f6379b39deb2beb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:37:49 +0000 Subject: [PATCH 2/2] ci: run the full matrix once in the merge queue and timing tiers nightly A passing fast core reports as Verify when the repository variable SQLBUILD_MERGE_QUEUE is true, because the merge queue runs the full matrix on every group before it merges. A merge group whose whole diff is release metadata skips the suites, as release pull requests already do. Pull requests and merge groups run the 3k-model compile jobs and the 1k dense job; a nightly scheduled run on main adds 5k and 10k and every wheel platform, and opens or updates an issue when it fails. Changes to the orchestration and CLI Rust test crates run native checks and the scheduler recovery suite instead of every suite. --- .github/workflows/verify.yml | 103 ++++++++++++++++++++++++++++++----- CONTRIBUTING.md | 13 +++-- 2 files changed, 97 insertions(+), 19 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 1170454220..74401885ca 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -5,6 +5,10 @@ on: # Labelling a pull request `full-verify` (or removing the label) changes how much it runs. types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] merge_group: + # Nightly, main also runs the 5k and 10k-model compile jobs that pull requests and merge groups + # leave out, and opens or updates an issue when it fails. + schedule: + - cron: "17 3 * * *" workflow_dispatch: permissions: @@ -13,12 +17,14 @@ permissions: # Each merge group has its own gh-readonly-queue ref, so queued groups never cancel each other; the # queue itself discards a group's run when it rebuilds the group. A push to a pull request cancels # its older run. Adding or removing any label other than `full-verify` starts a run in a group of its -# own that verifies nothing, so it never cancels the pull request's real run. +# own that verifies nothing, so it never cancels the pull request's real run. The nightly run has +# its own group, so a manual run on main and the nightly never cancel each other. concurrency: group: >- verify-${{ github.workflow }}-${{ github.event.merge_group.head_ref || github.ref }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') - && github.event.label.name != 'full-verify' && format('-label-{0}', github.run_id) || '' }} + && github.event.label.name != 'full-verify' && format('-label-{0}', github.run_id) || '' }}${{ + github.event_name == 'schedule' && '-nightly' || '' }} cancel-in-progress: ${{ github.event_name != 'merge_group' }} jobs: @@ -41,6 +47,9 @@ jobs: wheels: ${{ steps.scope.outputs.wheels }} recovery: ${{ steps.scope.outputs.recovery }} ui: ${{ steps.scope.outputs.ui }} + cold_models: ${{ steps.tiers.outputs.cold_models }} + dense_models: ${{ steps.tiers.outputs.dense_models }} + cache_models: ${{ steps.tiers.outputs.cache_models }} steps: - uses: actions/checkout@v4 with: @@ -124,10 +133,13 @@ jobs: # required Verify status is posted only by the trusted release checks in # release-please.yml after the metadata, version and distribution checks pass; the # verify job below reports under another name on Release Please branches. - if [[ "${EVENT_NAME}" == "pull_request" \ + # A merge group is judged by its content alone: one whose whole diff is release metadata + # carries only a release pull request whose product code already passed on main. + if { [[ "${EVENT_NAME}" == "merge_group" ]] \ + || [[ "${EVENT_NAME}" == "pull_request" \ && "${HEAD_REPOSITORY}" == "${GITHUB_REPOSITORY}" \ - && "${HEAD_REF}" == release-please--branches--* ]] && release_metadata_only; then - echo "Release Please pull request changes only release metadata; skipping the suites" \ + && "${HEAD_REF}" == release-please--branches--* ]]; } && release_metadata_only; then + echo "The change is only release metadata; skipping the suites" \ | tee -a "${GITHUB_STEP_SUMMARY}" write_scope false false false false false false false false false false false exit 0 @@ -169,6 +181,9 @@ jobs: *) ui_only=false ;; esac case "${path}" in + # Rust test crates and modules that only `cargo test` compiles; they are not in the + # native build identity and never reach a wheel. + crates/sqlbuild-cli/tests/*|crates/sqlbuild-orchestration/tests/*) ;; crates/*|ui/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml|deploy/Dockerfile|deploy/project/*|deploy/compose*|compose.yml|.dockerignore|Makefile|tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_compose.py|tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/*|benchmarks/duckshop/edge_cases/orchestration/container_deployment.json) wheels=true ;; @@ -192,6 +207,10 @@ jobs: ;; esac case "${path}" in + crates/sqlbuild-cli/tests/*|crates/sqlbuild-orchestration/tests/*) + # Native checks and the scheduler recovery suite run these tests; the Python, + # warehouse, wheel and timing suites cannot see them. + ;; crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock) broad=true performance=true @@ -278,12 +297,33 @@ jobs: "${recovery}" \ "${ui}" + # Pull requests and merge groups run the 3k-model compile jobs and the 1k dense job; the + # nightly run and manual runs add 5k and 10k. Budgets and ratio gates are the same in both. + - name: Choose the compile-performance model counts + id: tiers + env: + ALL: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }} + run: | + if [[ "${ALL}" == "true" ]]; then + echo "cold_models=[3000, 5000, 10000]" + echo "dense_models=[1000, 3000, 5000, 10000]" + echo "cache_models=[3000, 5000, 10000]" + else + echo "cold_models=[3000]" + echo "dense_models=[1000, 3000]" + echo "cache_models=[3000]" + fi >> "${GITHUB_OUTPUT}" + # Pull requests run the fast Linux core: static checks, unit and integration tests, native # workspace checks and the UI checks. The rest of the matrix (wheels, Compose, Windows, macOS, # DuckDB E2E, warehouse, compiler-golden and timing jobs) runs only once a pull request is # labelled `full-verify` and is not a draft, and always on merge groups and manual runs. A # run that leaves out suites its change needs reports under another name, so the required - # Verify stays missing and blocks the merge until the full matrix has passed. + # Verify stays missing and blocks the merge until the full matrix has passed. When the + # repository variable SQLBUILD_MERGE_QUEUE is true, the merge queue runs the full matrix on + # every group before it merges, so a passing fast core reports as Verify and each pull + # request runs the heavy suites once, in the queue. Unsetting the variable restores the old + # behaviour. - name: Choose the fast core or the full matrix id: depth env: @@ -297,7 +337,7 @@ jobs: partial=false if [[ "${FULL}" != "true" && "${HEAVY_SELECTED}" == "true" ]]; then partial=true - echo "Fast core only: label the pull request full-verify to run the full matrix Verify needs" \ + echo "Fast core only: the full-verify label, or the merge queue when it verifies groups, runs the full matrix" \ | tee -a "${GITHUB_STEP_SUMMARY}" fi { @@ -747,7 +787,7 @@ jobs: strategy: fail-fast: false matrix: - models: [3000, 5000, 10000] + models: ${{ fromJSON(needs.changes.outputs.cold_models) }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-uv @@ -799,7 +839,7 @@ jobs: strategy: fail-fast: false matrix: - models: [1000, 3000, 5000, 10000] + models: ${{ fromJSON(needs.changes.outputs.dense_models) }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-uv @@ -949,7 +989,7 @@ jobs: strategy: fail-fast: false matrix: - models: [3000, 5000, 10000] + models: ${{ fromJSON(needs.changes.outputs.cache_models) }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-uv @@ -1212,15 +1252,16 @@ jobs: # metadata-only pass can never satisfy Verify first; a skipped job would count as passing, so # the job always runs and only its name changes. Until those checks post Verify on the head, # the pull request waits for an expected check, and every new push to the branch waits again. - # The same holds for a fast-core run that leaves out suites its change needs, and for a run - # started by an unrelated label: neither reports as Verify. + # The same holds for a run started by an unrelated label, and for a fast-core run that leaves + # out suites its change needs, unless the merge queue (SQLBUILD_MERGE_QUEUE) runs them. name: >- ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && startsWith(github.event.pull_request.head.ref, 'release-please--branches--')) && 'Release pull request suites' || ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'full-verify') && 'Label change (nothing verified)' - || needs.changes.outputs.partial == 'true' && 'Fast core (label full-verify for Verify)' + || needs.changes.outputs.partial == 'true' && vars.SQLBUILD_MERGE_QUEUE != 'true' + && 'Fast core (label full-verify for Verify)' || 'Verify' }} if: always() needs: @@ -1354,7 +1395,41 @@ jobs: fi done else - echo "Fast core passed; the full matrix runs once the pull request is labelled full-verify" + echo "Fast core passed; the full matrix runs on full-verify, or in the merge queue when it verifies groups" exit 0 fi echo "All path-relevant verification jobs passed" + + report_nightly: + name: Report nightly failure + needs: verify + if: ${{ !cancelled() && github.event_name == 'schedule' && needs.verify.result == 'failure' }} + runs-on: ubuntu-24.04 + permissions: + issues: write + steps: + - name: Open or update the nightly Verify issue + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + LABEL: nightly-verify + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + { + echo "Nightly Verify failed at ${GITHUB_SHA} on main." + echo + echo "Run: ${RUN_URL}" + echo + echo "The nightly run adds the 5k and 10k-model compile jobs and every wheel platform." + } > report.md + gh label create "${LABEL}" --color B60205 \ + --description "The nightly Verify run on main failed" 2>/dev/null || true + issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \ + --json number --jq '.[0].number // empty')" + if [[ -n "${issue}" ]]; then + gh issue comment "${issue}" --body-file report.md + echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}" + else + gh issue create --title "Nightly Verify failed on main" \ + --label "${LABEL}" --body-file report.md + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f7447e387..890b9a7327 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,11 +96,14 @@ make verify In CI, every push to a pull request runs a fast Linux core: static checks, unit and integration tests and native workspace checks, plus the UI checks for UI changes. A change confined to the web -UI and documentation runs only the UI checks. The full matrix (wheels, Compose, Windows, macOS, -DuckDB end-to-end, warehouse, compiler-golden and timing jobs) runs once a maintainer labels a -pull request that is not a draft `full-verify`, normally when it is next to merge. The required -`Verify` check only reports after every suite the change needs has passed, so a pull request that -needs the full matrix cannot merge until it has run. +UI and documentation runs only the UI checks. The full matrix (the Linux wheel and Compose, +Windows, macOS, DuckDB end-to-end, warehouse, compiler-golden and 3,000-model timing jobs) runs on +every merge group in the merge queue before the change merges. Label a pull request that is not a +draft `full-verify` to run it on the pull request itself; until the merge queue verifies groups +(`SQLBUILD_MERGE_QUEUE`), a change that needs the heavy suites reports `Verify` only after that +label's run. The 5,000 and 10,000-model +timing jobs and every wheel platform run nightly on `main`, and a failed nightly run opens or +updates a `nightly-verify` issue. Each release builds and checks every wheel platform. For focused iteration, run the narrowest relevant pytest target, for example: